From a25be5c95c56a42b4059663473fd7ca2a9b105c4 Mon Sep 17 00:00:00 2001 From: Matiss Janis Aboltins Date: Fri, 20 Feb 2026 18:01:36 +0000 Subject: [PATCH] [AI] Remove usage of 'web' file types (#7033) * [AI] Desktop client, E2E, loot-core, sync-server and tooling updates Co-authored-by: Cursor * Refactor database handling in various modules to use async/await for improved readability and error handling. This includes updates to database opening and closing methods across multiple files, ensuring consistent asynchronous behavior. Additionally, minor adjustments were made to encryption functions to support async operations. * Refactor sync migration tests to utilize async/await for improved readability. Updated transaction handling to streamline event expectations and cleanup process. * Refactor various functions to utilize async/await for improved readability and error handling. Updated service stopping, encryption, and file upload/download methods to ensure consistent asynchronous behavior across the application. * Refactor BudgetFileSelection component to use async/await for onSelect method, enhancing error handling and readability. Update merge tests to utilize async/await for improved clarity in transaction merging expectations. * Refactor filesystem module to use async/await for init function and related database operations, enhancing error handling and consistency across file interactions. Updated tests to reflect asynchronous behavior in database operations and file writing. * Fix typo in init function declaration to ensure it returns a Promise instead of Proise. * Update VRT screenshots Auto-generated by VRT workflow PR: #6987 * Update tests to use async/await for init function in web filesystem, ensuring consistent asynchronous behavior in database operations. * Update VRT screenshot for payees filter test to reflect recent changes * Update filesystem module to remove web-specific implementations and streamline path handling. Refactor file operations to enhance type safety and consistency across different environments. Add tests for SQLite interactions and ensure proper handling of database transactions. * Add release notes for maintenance: Remove usage of 'web' file types * Refactor filesystem module to use type annotations for exports and improve consistency across methods. Remove deprecated web file handling and enhance encryption functions for better browser compatibility. * Trigger CI * Add asyncStorage API file to export Electron index module * Trigger CI * Feedback: typo --------- Co-authored-by: Cursor Co-authored-by: github-actions[bot] --- .../component-library/vitest.web.config.ts | 1 - packages/desktop-client/vite.config.mts | 4 - packages/loot-core/package.json | 8 +- .../platform/server/asyncStorage/index.api.ts | 2 + .../platform/server/fetch/index.electron.ts | 7 +- .../src/platform/server/fs/index.api.ts | 2 + .../src/platform/server/fs/index.electron.ts | 71 +-- .../fs/{index.web.test.ts => index.test.ts} | 0 .../loot-core/src/platform/server/fs/index.ts | 443 +++++++++++++++--- .../src/platform/server/fs/index.web.ts | 396 ---------------- .../src/platform/server/fs/path-join.api.ts | 1 + .../src/platform/server/fs/path-join.ts | 99 +++- .../src/platform/server/fs/path-join.web.ts | 99 ---- .../src/platform/server/fs/shared.ts | 4 +- .../src/platform/server/sqlite/index.api.ts | 2 + .../{index.web.test.ts => index.test.ts} | 9 +- .../src/platform/server/sqlite/index.ts | 225 ++++++++- .../src/platform/server/sqlite/index.web.ts | 218 --------- packages/loot-core/src/server/db/index.ts | 12 +- .../encryption/encryption-internals.api.ts | 2 + .../encryption-internals.electron.ts | 89 ++++ .../server/encryption/encryption-internals.ts | 128 +++-- .../encryption/encryption-internals.web.ts | 109 ----- .../{platform.web.ts => platform.ts} | 0 .../loot-core/src/shared/platform.electron.ts | 10 +- packages/loot-core/src/shared/platform.ts | 23 +- packages/loot-core/src/shared/platform.web.ts | 20 - packages/loot-core/vite.api.config.ts | 3 - packages/loot-core/vite.config.ts | 10 +- packages/loot-core/vitest.config.ts | 6 +- packages/loot-core/vitest.web.config.ts | 6 +- tsconfig.json | 1 + upcoming-release-notes/7033.md | 6 + 33 files changed, 960 insertions(+), 1056 deletions(-) create mode 100644 packages/loot-core/src/platform/server/asyncStorage/index.api.ts create mode 100644 packages/loot-core/src/platform/server/fs/index.api.ts rename packages/loot-core/src/platform/server/fs/{index.web.test.ts => index.test.ts} (100%) delete mode 100644 packages/loot-core/src/platform/server/fs/index.web.ts create mode 100644 packages/loot-core/src/platform/server/fs/path-join.api.ts delete mode 100644 packages/loot-core/src/platform/server/fs/path-join.web.ts create mode 100644 packages/loot-core/src/platform/server/sqlite/index.api.ts rename packages/loot-core/src/platform/server/sqlite/{index.web.test.ts => index.test.ts} (96%) delete mode 100644 packages/loot-core/src/platform/server/sqlite/index.web.ts create mode 100644 packages/loot-core/src/server/encryption/encryption-internals.api.ts create mode 100644 packages/loot-core/src/server/encryption/encryption-internals.electron.ts delete mode 100644 packages/loot-core/src/server/encryption/encryption-internals.web.ts rename packages/loot-core/src/shared/__mocks__/{platform.web.ts => platform.ts} (100%) delete mode 100644 packages/loot-core/src/shared/platform.web.ts create mode 100644 upcoming-release-notes/7033.md diff --git a/packages/component-library/vitest.web.config.ts b/packages/component-library/vitest.web.config.ts index e985019ddf..01e3286952 100644 --- a/packages/component-library/vitest.web.config.ts +++ b/packages/component-library/vitest.web.config.ts @@ -5,7 +5,6 @@ import { defineConfig } from 'vitest/config'; const resolveExtensions = [ '.testing.ts', - '.web.ts', '.mjs', '.js', '.mts', diff --git a/packages/desktop-client/vite.config.mts b/packages/desktop-client/vite.config.mts index 0ee5663aeb..add2ccb40f 100644 --- a/packages/desktop-client/vite.config.mts +++ b/packages/desktop-client/vite.config.mts @@ -82,10 +82,6 @@ export default defineConfig(async ({ mode }) => { } let resolveExtensions = [ - '.web.js', - '.web.jsx', - '.web.ts', - '.web.tsx', '.mjs', '.js', '.mts', diff --git a/packages/loot-core/package.json b/packages/loot-core/package.json index 2fec1caf07..c983a249f6 100644 --- a/packages/loot-core/package.json +++ b/packages/loot-core/package.json @@ -40,10 +40,10 @@ "./platform/exceptions": "./src/platform/exceptions/index.ts", "./platform/server/asyncStorage": "./src/platform/server/asyncStorage/index.ts", "./platform/server/connection": "./src/platform/server/connection/index.ts", - "./platform/server/fetch": "./src/platform/server/fetch/index.web.ts", - "./platform/server/fs": "./src/platform/server/fs/index.web.ts", - "./platform/server/log": "./src/platform/server/log/index.web.ts", - "./platform/server/sqlite": "./src/platform/server/sqlite/index.web.ts", + "./platform/server/fetch": "./src/platform/server/fetch/index.ts", + "./platform/server/fs": "./src/platform/server/fs/index.ts", + "./platform/server/log": "./src/platform/server/log/index.ts", + "./platform/server/sqlite": "./src/platform/server/sqlite/index.ts", "./server/budget/types/*": "./src/server/budget/types/*.d.ts", "./server/*": "./src/server/*.ts", "./shared/*": "./src/shared/*.ts", diff --git a/packages/loot-core/src/platform/server/asyncStorage/index.api.ts b/packages/loot-core/src/platform/server/asyncStorage/index.api.ts new file mode 100644 index 0000000000..1498cc1767 --- /dev/null +++ b/packages/loot-core/src/platform/server/asyncStorage/index.api.ts @@ -0,0 +1,2 @@ +// oxlint-disable-next-line no-restricted-imports +export * from './index.electron'; diff --git a/packages/loot-core/src/platform/server/fetch/index.electron.ts b/packages/loot-core/src/platform/server/fetch/index.electron.ts index 34fbbb146e..41b985bb60 100644 --- a/packages/loot-core/src/platform/server/fetch/index.electron.ts +++ b/packages/loot-core/src/platform/server/fetch/index.electron.ts @@ -1,9 +1,8 @@ import { logger } from '../log'; -export const fetch = async ( - input: RequestInfo | URL, - options?: RequestInit, -) => { +import type * as T from './index'; + +export const fetch: typeof T.fetch = async (input, options) => { try { return await globalThis.fetch(input, { ...options, diff --git a/packages/loot-core/src/platform/server/fs/index.api.ts b/packages/loot-core/src/platform/server/fs/index.api.ts new file mode 100644 index 0000000000..1498cc1767 --- /dev/null +++ b/packages/loot-core/src/platform/server/fs/index.api.ts @@ -0,0 +1,2 @@ +// oxlint-disable-next-line no-restricted-imports +export * from './index.electron'; diff --git a/packages/loot-core/src/platform/server/fs/index.electron.ts b/packages/loot-core/src/platform/server/fs/index.electron.ts index 67fdaaa6b7..e0f5798014 100644 --- a/packages/loot-core/src/platform/server/fs/index.electron.ts +++ b/packages/loot-core/src/platform/server/fs/index.electron.ts @@ -6,7 +6,7 @@ import promiseRetry from 'promise-retry'; import { logger } from '../log'; -import type * as T from '.'; +import type * as T from './index'; export { getDocumentDir, getBudgetDir, _setDocumentDir } from './shared'; @@ -23,28 +23,37 @@ switch (path.basename(__filename)) { break; } -export const init = async () => { +export const init: typeof T.init = async () => { // Nothing to do }; -export const getDataDir = () => { +export const getDataDir: typeof T.getDataDir = () => { if (!process.env.ACTUAL_DATA_DIR) { throw new Error('ACTUAL_DATA_DIR env variable is required'); } return process.env.ACTUAL_DATA_DIR; }; -export const bundledDatabasePath = path.join(rootPath, 'default-db.sqlite'); +export const bundledDatabasePath: typeof T.bundledDatabasePath = path.join( + rootPath, + 'default-db.sqlite', +); -export const migrationsPath = path.join(rootPath, 'migrations'); +export const migrationsPath: typeof T.migrationsPath = path.join( + rootPath, + 'migrations', +); -export const demoBudgetPath = path.join(rootPath, 'demo-budget'); +export const demoBudgetPath: typeof T.demoBudgetPath = path.join( + rootPath, + 'demo-budget', +); -export const join = path.join; +export const join: typeof T.join = path.join; -export const basename = filepath => path.basename(filepath); +export const basename: typeof T.basename = filepath => path.basename(filepath); -export const listDir: T.ListDir = filepath => +export const listDir: typeof T.listDir = filepath => new Promise((resolve, reject) => { fs.readdir(filepath, (err, files) => { if (err) { @@ -55,14 +64,14 @@ export const listDir: T.ListDir = filepath => }); }); -export const exists = filepath => +export const exists: typeof T.exists = filepath => new Promise(resolve => { fs.access(filepath, fs.constants.F_OK, err => { return resolve(!err); }); }); -export const mkdir = filepath => +export const mkdir: typeof T.mkdir = filepath => new Promise((resolve, reject) => { fs.mkdir(filepath, err => { if (err) { @@ -73,7 +82,7 @@ export const mkdir = filepath => }); }); -export const size = filepath => +export const size: typeof T.size = filepath => new Promise((resolve, reject) => { fs.stat(filepath, (err, stats) => { if (err) { @@ -84,7 +93,7 @@ export const size = filepath => }); }); -export const copyFile: T.CopyFile = (frompath, topath) => { +export const copyFile: typeof T.copyFile = (frompath, topath) => { return new Promise((resolve, reject) => { const readStream = fs.createReadStream(frompath); const writeStream = fs.createWriteStream(topath); @@ -97,7 +106,7 @@ export const copyFile: T.CopyFile = (frompath, topath) => { }); }; -export const readFile: T.ReadFile = ( +export const readFile: typeof T.readFile = ( filepath: string, encoding: 'utf8' | 'binary' | null = 'utf8', ) => { @@ -119,12 +128,11 @@ export const readFile: T.ReadFile = ( }); }; -export const writeFile: T.WriteFile = async (filepath, contents) => { +export const writeFile: typeof T.writeFile = async (filepath, contents) => { try { await promiseRetry( (retry, attempt) => { return new Promise((resolve, reject) => { - // @ts-expect-error contents type needs refining fs.writeFile(filepath, contents, 'utf8', err => { if (err) { logger.error( @@ -157,7 +165,7 @@ export const writeFile: T.WriteFile = async (filepath, contents) => { } }; -export const removeFile = filepath => { +export const removeFile: typeof T.removeFile = filepath => { return new Promise(function (resolve, reject) { fs.unlink(filepath, err => { return err ? reject(err) : resolve(undefined); @@ -165,7 +173,7 @@ export const removeFile = filepath => { }); }; -export const removeDir = dirpath => { +export const removeDir: typeof T.removeDir = dirpath => { return new Promise(function (resolve, reject) { fs.rmdir(dirpath, err => { return err ? reject(err) : resolve(undefined); @@ -173,22 +181,23 @@ export const removeDir = dirpath => { }); }; -export const removeDirRecursively = async dirpath => { - if (await exists(dirpath)) { - for (const file of await listDir(dirpath)) { - const fullpath = join(dirpath, file); - if (fs.statSync(fullpath).isDirectory()) { - await removeDirRecursively(fullpath); - } else { - await removeFile(fullpath); +export const removeDirRecursively: typeof T.removeDirRecursively = + async dirpath => { + if (await exists(dirpath)) { + for (const file of await listDir(dirpath)) { + const fullpath = join(dirpath, file); + if (fs.statSync(fullpath).isDirectory()) { + await removeDirRecursively(fullpath); + } else { + await removeFile(fullpath); + } } + + await removeDir(dirpath); } + }; - await removeDir(dirpath); - } -}; - -export const getModifiedTime = filepath => { +export const getModifiedTime: typeof T.getModifiedTime = filepath => { return new Promise(function (resolve, reject) { fs.stat(filepath, (err, stats) => { if (err) { diff --git a/packages/loot-core/src/platform/server/fs/index.web.test.ts b/packages/loot-core/src/platform/server/fs/index.test.ts similarity index 100% rename from packages/loot-core/src/platform/server/fs/index.web.test.ts rename to packages/loot-core/src/platform/server/fs/index.test.ts diff --git a/packages/loot-core/src/platform/server/fs/index.ts b/packages/loot-core/src/platform/server/fs/index.ts index 60407ef88d..7608228fbd 100644 --- a/packages/loot-core/src/platform/server/fs/index.ts +++ b/packages/loot-core/src/platform/server/fs/index.ts @@ -1,79 +1,416 @@ -export { join } from './path-join'; +// @ts-strict-ignore +import { SQLiteFS } from 'absurd-sql'; +import IndexedDBBackend from 'absurd-sql/dist/indexeddb-backend'; -export declare function init(): Promise; -export type Init = typeof init; +import * as connection from '../connection'; +import * as idb from '../indexeddb'; +import { logger } from '../log'; +import { _getModule } from '../sqlite'; +import type { SqlJsModule } from '../sqlite'; -export declare function getDataDir(): string; -export type GetDataDir = typeof getDataDir; +import { join } from './path-join'; -export declare function _setDocumentDir(dir: string): string; -export type _SetDocumentDir = typeof _setDocumentDir; +let FS: SqlJsModule['FS'] = null; +let BFS = null; +const NO_PERSIST = false; -export declare function getDocumentDir(): string; -export type GetDocumentDir = typeof getDocumentDir; +export const bundledDatabasePath: string = '/default-db.sqlite'; +export const migrationsPath: string = '/migrations'; +export const demoBudgetPath: string = '/demo-budget'; +export { join }; +export { getDocumentDir, getBudgetDir, _setDocumentDir } from './shared'; +export const getDataDir = () => process.env.ACTUAL_DATA_DIR; -export declare function getBudgetDir(id: string): string; -export type GetBudgetDir = typeof getBudgetDir; +export const pathToId = function (filepath: string): string { + return filepath.replace(/^\//, '').replace(/\//g, '-'); +}; -export declare const bundledDatabasePath: string; -export type BundledDatabasePath = typeof bundledDatabasePath; +function _exists(filepath: string): boolean { + try { + FS.readlink(filepath); + return true; + } catch {} -export declare const migrationsPath: string; -export type MigrationsPath = typeof migrationsPath; + try { + FS.stat(filepath); + return true; + } catch {} + return false; +} -export declare const demoBudgetPath: string; -export type DemoBudgetPath = typeof demoBudgetPath; +function _mkdirRecursively(dir) { + const parts = dir.split('/').filter(str => str !== ''); + let path = ''; + for (const part of parts) { + path += '/' + part; + if (!_exists(path)) { + FS.mkdir(path); + } + } +} -export declare function pathToId(filepath: string): string; -export type PathToId = typeof pathToId; +function _createFile(filepath: string) { + // This can create the file. Check if it exists, if not create a + // symlink if it's a sqlite file. Otherwise store in idb -export declare function basename(filepath: string): string; -export type Basename = typeof basename; + if (!NO_PERSIST && filepath.startsWith('/documents')) { + if (filepath.endsWith('.sqlite')) { + // If it doesn't exist, we need to create a symlink + if (!_exists(filepath)) { + FS.symlink('/blocked/' + pathToId(filepath), filepath); + } + } else { + // The contents are actually stored in IndexedDB. We only write to + // the in-memory fs to take advantage of the file hierarchy + FS.writeFile(filepath, '!$@) this should never read !$@)'); + } + } -export declare function listDir(filepath: string): Promise; -export type ListDir = typeof listDir; + return filepath; +} -export declare function exists(filepath: string): Promise; -export type Exists = typeof exists; +async function _readFile( + filepath: string, + opts: { encoding: 'utf8' }, +): Promise; +async function _readFile( + filepath: string, + opts?: { encoding: 'binary' }, +): Promise; +async function _readFile( + filepath: string, + opts?: { encoding: 'utf8' } | { encoding: 'binary' }, +): Promise { + // We persist stuff in /documents, but don't need to handle sqlite + // file specifically because those are symlinked to a separate + // filesystem and will be handled in the BlockedFS + if ( + !NO_PERSIST && + filepath.startsWith('/documents') && + !filepath.endsWith('.sqlite') + ) { + if (!_exists(filepath)) { + throw new Error('File does not exist: ' + filepath); + } -export declare function mkdir(filepath: string): Promise; -export type Mkdir = typeof mkdir; + // Grab contents from IDB + const { store } = idb.getStore(await idb.getDatabase(), 'files'); + const item = await idb.get(store, filepath); -export declare function size(filepath: string): Promise; -export type Size = typeof size; + if (item == null) { + throw new Error('File does not exist: ' + filepath); + } -export declare function copyFile( + if (opts?.encoding === 'utf8' && ArrayBuffer.isView(item.contents)) { + return String.fromCharCode.apply( + null, + new Uint16Array(item.contents.buffer), + ); + } + + return item.contents; + } else { + if (opts?.encoding === 'utf8') { + return FS.readFile(resolveLink(filepath), { encoding: 'utf8' }); + } else if (opts?.encoding === 'binary') { + return FS.readFile(resolveLink(filepath), { encoding: 'binary' }); + } else { + return FS.readFile(resolveLink(filepath)); + } + } +} + +function resolveLink(path: string): string { + try { + const { node } = FS.lookupPath(path, { follow: false }); + return node.link ? FS.readlink(path) : path; + } catch { + return path; + } +} + +async function _writeFile(filepath: string, contents): Promise { + if (contents instanceof ArrayBuffer) { + contents = new Uint8Array(contents); + } else if (ArrayBuffer.isView(contents)) { + contents = new Uint8Array(contents.buffer); + } + + // We always create the file if it doesn't exist, and this function + // setups up the file depending on its type + _createFile(filepath); + + if (!NO_PERSIST && filepath.startsWith('/documents')) { + const isDb = filepath.endsWith('.sqlite'); + + // Write to IDB + const { store } = idb.getStore(await idb.getDatabase(), 'files'); + + if (isDb) { + // We never write the contents of the database to idb ourselves. + // It gets handled via a symlink to the blocked fs (created by + // `_createFile` above). However, we still need to record an + // entry for the db file so the fs gets properly constructed on + // startup + await idb.set(store, { filepath, contents: '' }); + + // Actually persist the data by going the FS, which will pass + // the data through the symlink to the blocked fs. For some + // reason we need to resolve symlinks ourselves. + await Promise.resolve(); + FS.writeFile(resolveLink(filepath), contents); + } else { + await idb.set(store, { filepath, contents }); + } + } else { + FS.writeFile(resolveLink(filepath), contents); + } + return true; +} + +async function _copySqlFile( frompath: string, topath: string, -): Promise; -export type CopyFile = typeof copyFile; +): Promise { + _createFile(topath); -export declare function readFile( - filepath: string, - encoding: 'binary' | null, -): Promise; -export declare function readFile( + const { store } = idb.getStore(await idb.getDatabase(), 'files'); + await idb.set(store, { filepath: topath, contents: '' }); + const fromitem = await idb.get(store, frompath); + const fromDbPath = pathToId(fromitem.filepath); + const toDbPath = pathToId(topath); + + const fromfile = BFS.backend.createFile(fromDbPath); + const tofile = BFS.backend.createFile(toDbPath); + + try { + fromfile.open(); + tofile.open(); + const fileSize = fromfile.meta.size; + const blockSize = fromfile.meta.blockSize; + + const buffer = new ArrayBuffer(blockSize); + const bufferView = new Uint8Array(buffer); + + for (let i = 0; i < fileSize; i += blockSize) { + const bytesToRead = Math.min(blockSize, fileSize - i); + fromfile.read(bufferView, 0, bytesToRead, i); + tofile.write(bufferView, 0, bytesToRead, i); + } + } catch (error) { + tofile.close(); + fromfile.close(); + await _removeFile(toDbPath); + logger.error('Failed to copy database file', error); + return false; + } finally { + tofile.close(); + fromfile.close(); + } + + return true; +} + +async function _removeFile(filepath: string) { + if (!NO_PERSIST && filepath.startsWith('/documents')) { + const isDb = filepath.endsWith('.sqlite'); + + // Remove from IDB + const { store } = idb.getStore(await idb.getDatabase(), 'files'); + await idb.del(store, filepath); + + // If this is the database, is has been symlinked and we want to + // remove the actual contents + if (isDb) { + const linked = resolveLink(filepath); + // Be resilient to fs corruption: don't throw an error by trying + // to remove a file that doesn't exist. For some reason the db + // file is gone? It's ok, just ignore it + if (_exists(linked)) { + FS.unlink(linked); + } + } + } + + // Finally, remove any in-memory instance + FS.unlink(filepath); +} + +// Load files from the server that should exist by default +async function populateDefaultFilesystem() { + const index = await ( + await fetch(process.env.PUBLIC_URL + 'data-file-index.txt') + ).text(); + const files = index + .split('\n') + .map(name => name.trim()) + .filter(name => name !== ''); + const fetchFile = url => fetch(url).then(res => res.arrayBuffer()); + + // This is hardcoded. We know we must create the migrations + // directory, it's not worth complicating the index to support + // creating arbitrary folders. + await mkdir('/migrations'); + await mkdir('/demo-budget'); + + await Promise.all( + files.map(async file => { + const contents = await fetchFile(process.env.PUBLIC_URL + 'data/' + file); + await _writeFile('/' + file, contents); + }), + ); +} + +const populateFileHierarchy = async function () { + const { store } = idb.getStore(await idb.getDatabase(), 'files'); + const req = store.getAllKeys(); + const paths: string[] = await new Promise((resolve, reject) => { + // @ts-expect-error fix me + req.onsuccess = e => resolve(e.target.result); + req.onerror = e => reject(e); + }); + + for (const path of paths) { + _mkdirRecursively(basename(path)); + _createFile(path); + } +}; + +export const init = async function () { + const Module = _getModule(); + FS = Module.FS; + + // When a user "uploads" a file, we just put it in memory in this + // dir and the backend takes it from there + FS.mkdir('/uploads'); + + // Files in /documents are actually read/written from idb. + // Everything in there is automatically persisted + FS.mkdir('/documents'); + + // Files in /blocked are handled by the BlockedFS, which is a + // special fs that persists files in blocks. This is necessary + // for sqlite3 + FS.mkdir('/blocked'); + + // Jest doesn't support workers. Right now we disable the blocked fs + // backend under testing and just test that the directory structure + // is created correctly. We assume the the absurd-sql project tests + // the blocked fs enough. Additionally, we don't populate the + // default files in testing. + if (process.env.NODE_ENV !== 'test') { + const backend = new IndexedDBBackend(() => { + connection.send('fallback-write-error'); + }); + BFS = new SQLiteFS(FS, backend); + Module.register_for_idb(BFS); + + FS.mount(BFS, {}, '/blocked'); + + await populateDefaultFilesystem(); + } + + await populateFileHierarchy(); +}; + +export const basename = function (filepath) { + const parts = filepath.split('/'); + return parts.slice(0, -1).join('/'); +}; + +export const listDir = async function (filepath) { + const paths = FS.readdir(filepath); + return paths.filter(p => p !== '.' && p !== '..'); +}; + +export const exists = async function (filepath) { + return _exists(filepath); +}; + +export const mkdir = async function (filepath) { + FS.mkdir(filepath); +}; + +export const size = async function (filepath) { + const attrs = FS.stat(resolveLink(filepath)); + return attrs.size; +}; + +export const copyFile = async function ( + frompath: string, + topath: string, +): Promise { + let result = false; + try { + const contents = await _readFile(frompath); + result = await _writeFile(topath, contents); + } catch (error) { + if (frompath.endsWith('.sqlite') || topath.endsWith('.sqlite')) { + try { + result = await _copySqlFile(frompath, topath); + } catch (secondError) { + throw new Error( + `Failed to copy SQL file from ${frompath} to ${topath}: ${secondError.message}`, + ); + } + } else { + throw error; + } + } + return result; +}; + +export async function readFile( filepath: string, encoding?: 'utf8', ): Promise; -export type ReadFile = typeof readFile; - -export declare function writeFile( +export async function readFile( filepath: string, - contents: string | ArrayBuffer | NodeJS.ArrayBufferView, -): Promise; -export type WriteFile = typeof writeFile; + encoding: 'binary', +): Promise; +export async function readFile( + filepath: string, + encoding: 'binary' | 'utf8' = 'utf8', +) { + if (encoding === 'utf8') { + return _readFile(filepath, { encoding }); + } -export declare function removeFile(filepath: string): Promise; -export type RemoveFile = typeof removeFile; + return _readFile(filepath, { encoding }); +} -export declare function removeDir(dirpath: string): Promise; -export type RemoveDir = typeof removeDir; +export const writeFile = async function (filepath: string, contents) { + return _writeFile(filepath, contents); +}; -export declare function removeDirRecursively( - dirpath: string, -): Promise; -export type RemoveDirRecursively = typeof removeDirRecursively; +export const removeFile = async function (filepath: string) { + return _removeFile(filepath); +}; -export declare function getModifiedTime(filepath: string): Promise; -export type GetModifiedTime = typeof getModifiedTime; +export const removeDir = async function (filepath) { + FS.rmdir(filepath); +}; + +export const removeDirRecursively = async function (dirpath) { + if (await exists(dirpath)) { + for (const file of await listDir(dirpath)) { + const fullpath = join(dirpath, file); + // `true` here means to not follow symlinks + const attr = FS.stat(fullpath, true); + + if (FS.isDir(attr.mode)) { + await removeDirRecursively(fullpath); + } else { + await removeFile(fullpath); + } + } + + await removeDir(dirpath); + } +}; + +export const getModifiedTime = async (filepath: string): Promise => { + throw new Error( + 'getModifiedTime not supported on the web (only used for backups)', + ); +}; diff --git a/packages/loot-core/src/platform/server/fs/index.web.ts b/packages/loot-core/src/platform/server/fs/index.web.ts deleted file mode 100644 index 85132058d3..0000000000 --- a/packages/loot-core/src/platform/server/fs/index.web.ts +++ /dev/null @@ -1,396 +0,0 @@ -// @ts-strict-ignore -import { SQLiteFS } from 'absurd-sql'; -import IndexedDBBackend from 'absurd-sql/dist/indexeddb-backend'; - -import * as connection from '../connection'; -import * as idb from '../indexeddb'; -import { logger } from '../log'; -import { _getModule } from '../sqlite'; -import type { SqlJsModule } from '../sqlite'; - -import { join } from './path-join'; - -let FS: SqlJsModule['FS'] = null; -let BFS = null; -const NO_PERSIST = false; - -export const bundledDatabasePath = '/default-db.sqlite'; -export const migrationsPath = '/migrations'; -export const demoBudgetPath = '/demo-budget'; -export { join }; -export { getDocumentDir, getBudgetDir, _setDocumentDir } from './shared'; -export const getDataDir = () => process.env.ACTUAL_DATA_DIR; - -export const pathToId = function (filepath: string): string { - return filepath.replace(/^\//, '').replace(/\//g, '-'); -}; - -function _exists(filepath: string): boolean { - try { - FS.readlink(filepath); - return true; - } catch {} - - try { - FS.stat(filepath); - return true; - } catch {} - return false; -} - -function _mkdirRecursively(dir) { - const parts = dir.split('/').filter(str => str !== ''); - let path = ''; - for (const part of parts) { - path += '/' + part; - if (!_exists(path)) { - FS.mkdir(path); - } - } -} - -function _createFile(filepath: string) { - // This can create the file. Check if it exists, if not create a - // symlink if it's a sqlite file. Otherwise store in idb - - if (!NO_PERSIST && filepath.startsWith('/documents')) { - if (filepath.endsWith('.sqlite')) { - // If it doesn't exist, we need to create a symlink - if (!_exists(filepath)) { - FS.symlink('/blocked/' + pathToId(filepath), filepath); - } - } else { - // The contents are actually stored in IndexedDB. We only write to - // the in-memory fs to take advantage of the file hierarchy - FS.writeFile(filepath, '!$@) this should never read !$@)'); - } - } - - return filepath; -} - -async function _readFile( - filepath: string, - opts?: { encoding: 'utf8' } | { encoding: 'binary' }, -): Promise { - // We persist stuff in /documents, but don't need to handle sqlite - // file specifically because those are symlinked to a separate - // filesystem and will be handled in the BlockedFS - if ( - !NO_PERSIST && - filepath.startsWith('/documents') && - !filepath.endsWith('.sqlite') - ) { - if (!_exists(filepath)) { - throw new Error('File does not exist: ' + filepath); - } - - // Grab contents from IDB - const { store } = idb.getStore(await idb.getDatabase(), 'files'); - const item = await idb.get(store, filepath); - - if (item == null) { - throw new Error('File does not exist: ' + filepath); - } - - if (opts?.encoding === 'utf8' && ArrayBuffer.isView(item.contents)) { - return String.fromCharCode.apply( - null, - new Uint16Array(item.contents.buffer), - ); - } - - return item.contents; - } else { - if (opts?.encoding === 'utf8') { - return FS.readFile(resolveLink(filepath), { encoding: 'utf8' }); - } else if (opts?.encoding === 'binary') { - return FS.readFile(resolveLink(filepath), { encoding: 'binary' }); - } else { - return FS.readFile(resolveLink(filepath)); - } - } -} - -function resolveLink(path: string): string { - try { - const { node } = FS.lookupPath(path, { follow: false }); - return node.link ? FS.readlink(path) : path; - } catch { - return path; - } -} - -async function _writeFile(filepath: string, contents): Promise { - if (contents instanceof ArrayBuffer) { - contents = new Uint8Array(contents); - } else if (ArrayBuffer.isView(contents)) { - contents = new Uint8Array(contents.buffer); - } - - // We always create the file if it doesn't exist, and this function - // setups up the file depending on its type - _createFile(filepath); - - if (!NO_PERSIST && filepath.startsWith('/documents')) { - const isDb = filepath.endsWith('.sqlite'); - - // Write to IDB - const { store } = idb.getStore(await idb.getDatabase(), 'files'); - - if (isDb) { - // We never write the contents of the database to idb ourselves. - // It gets handled via a symlink to the blocked fs (created by - // `_createFile` above). However, we still need to record an - // entry for the db file so the fs gets properly constructed on - // startup - await idb.set(store, { filepath, contents: '' }); - - // Actually persist the data by going the FS, which will pass - // the data through the symlink to the blocked fs. For some - // reason we need to resolve symlinks ourselves. - await Promise.resolve(); - FS.writeFile(resolveLink(filepath), contents); - } else { - await idb.set(store, { filepath, contents }); - } - } else { - FS.writeFile(resolveLink(filepath), contents); - } - return true; -} - -async function _copySqlFile( - frompath: string, - topath: string, -): Promise { - _createFile(topath); - - const { store } = idb.getStore(await idb.getDatabase(), 'files'); - await idb.set(store, { filepath: topath, contents: '' }); - const fromitem = await idb.get(store, frompath); - const fromDbPath = pathToId(fromitem.filepath); - const toDbPath = pathToId(topath); - - const fromfile = BFS.backend.createFile(fromDbPath); - const tofile = BFS.backend.createFile(toDbPath); - - try { - fromfile.open(); - tofile.open(); - const fileSize = fromfile.meta.size; - const blockSize = fromfile.meta.blockSize; - - const buffer = new ArrayBuffer(blockSize); - const bufferView = new Uint8Array(buffer); - - for (let i = 0; i < fileSize; i += blockSize) { - const bytesToRead = Math.min(blockSize, fileSize - i); - fromfile.read(bufferView, 0, bytesToRead, i); - tofile.write(bufferView, 0, bytesToRead, i); - } - } catch (error) { - tofile.close(); - fromfile.close(); - await _removeFile(toDbPath); - logger.error('Failed to copy database file', error); - return false; - } finally { - tofile.close(); - fromfile.close(); - } - - return true; -} - -async function _removeFile(filepath: string) { - if (!NO_PERSIST && filepath.startsWith('/documents')) { - const isDb = filepath.endsWith('.sqlite'); - - // Remove from IDB - const { store } = idb.getStore(await idb.getDatabase(), 'files'); - await idb.del(store, filepath); - - // If this is the database, is has been symlinked and we want to - // remove the actual contents - if (isDb) { - const linked = resolveLink(filepath); - // Be resilient to fs corruption: don't throw an error by trying - // to remove a file that doesn't exist. For some reason the db - // file is gone? It's ok, just ignore it - if (_exists(linked)) { - FS.unlink(linked); - } - } - } - - // Finally, remove any in-memory instance - FS.unlink(filepath); -} - -// Load files from the server that should exist by default -async function populateDefaultFilesystem() { - const index = await ( - await fetch(process.env.PUBLIC_URL + 'data-file-index.txt') - ).text(); - const files = index - .split('\n') - .map(name => name.trim()) - .filter(name => name !== ''); - const fetchFile = url => fetch(url).then(res => res.arrayBuffer()); - - // This is hardcoded. We know we must create the migrations - // directory, it's not worth complicating the index to support - // creating arbitrary folders. - await mkdir('/migrations'); - await mkdir('/demo-budget'); - - await Promise.all( - files.map(async file => { - const contents = await fetchFile(process.env.PUBLIC_URL + 'data/' + file); - await _writeFile('/' + file, contents); - }), - ); -} - -export const populateFileHeirarchy = async function () { - const { store } = idb.getStore(await idb.getDatabase(), 'files'); - const req = store.getAllKeys(); - const paths: string[] = await new Promise((resolve, reject) => { - // @ts-expect-error fix me - req.onsuccess = e => resolve(e.target.result); - req.onerror = e => reject(e); - }); - - for (const path of paths) { - _mkdirRecursively(basename(path)); - _createFile(path); - } -}; - -export const init = async function () { - const Module = _getModule(); - FS = Module.FS; - - // When a user "uploads" a file, we just put it in memory in this - // dir and the backend takes it from there - FS.mkdir('/uploads'); - - // Files in /documents are actually read/written from idb. - // Everything in there is automatically persisted - FS.mkdir('/documents'); - - // Files in /blocked are handled by the BlockedFS, which is a - // special fs that persists files in blocks. This is necessary - // for sqlite3 - FS.mkdir('/blocked'); - - // Jest doesn't support workers. Right now we disable the blocked fs - // backend under testing and just test that the directory structure - // is created correctly. We assume the the absurd-sql project tests - // the blocked fs enough. Additionally, we don't populate the - // default files in testing. - if (process.env.NODE_ENV !== 'test') { - const backend = new IndexedDBBackend(() => { - connection.send('fallback-write-error'); - }); - BFS = new SQLiteFS(FS, backend); - Module.register_for_idb(BFS); - - FS.mount(BFS, {}, '/blocked'); - - await populateDefaultFilesystem(); - } - - await populateFileHeirarchy(); -}; - -export const basename = function (filepath) { - const parts = filepath.split('/'); - return parts.slice(0, -1).join('/'); -}; - -export const listDir = async function (filepath) { - const paths = FS.readdir(filepath); - return paths.filter(p => p !== '.' && p !== '..'); -}; - -export const exists = async function (filepath) { - return _exists(filepath); -}; - -export const mkdir = async function (filepath) { - FS.mkdir(filepath); -}; - -export const size = async function (filepath) { - const attrs = FS.stat(resolveLink(filepath)); - return attrs.size; -}; - -export const copyFile = async function ( - frompath: string, - topath: string, -): Promise { - let result = false; - try { - const contents = await _readFile(frompath); - result = await _writeFile(topath, contents); - } catch (error) { - if (frompath.endsWith('.sqlite') || topath.endsWith('.sqlite')) { - try { - result = await _copySqlFile(frompath, topath); - } catch (secondError) { - throw new Error( - `Failed to copy SQL file from ${frompath} to ${topath}: ${secondError.message}`, - ); - } - } else { - throw error; - } - } - return result; -}; - -export const readFile = async function ( - filepath: string, - encoding: 'binary' | 'utf8' = 'utf8', -) { - return _readFile(filepath, { encoding }); -}; - -export const writeFile = async function (filepath: string, contents) { - return _writeFile(filepath, contents); -}; - -export const removeFile = async function (filepath: string) { - return _removeFile(filepath); -}; - -export const removeDir = async function (filepath) { - FS.rmdir(filepath); -}; - -export const removeDirRecursively = async function (dirpath) { - if (await exists(dirpath)) { - for (const file of await listDir(dirpath)) { - const fullpath = join(dirpath, file); - // `true` here means to not follow symlinks - const attr = FS.stat(fullpath, true); - - if (FS.isDir(attr.mode)) { - await removeDirRecursively(fullpath); - } else { - await removeFile(fullpath); - } - } - - await removeDir(dirpath); - } -}; - -export const getModifiedTime = async function () { - throw new Error( - 'getModifiedTime not supported on the web (only used for backups)', - ); -}; diff --git a/packages/loot-core/src/platform/server/fs/path-join.api.ts b/packages/loot-core/src/platform/server/fs/path-join.api.ts new file mode 100644 index 0000000000..911858b8d2 --- /dev/null +++ b/packages/loot-core/src/platform/server/fs/path-join.api.ts @@ -0,0 +1 @@ +export { join } from 'path'; diff --git a/packages/loot-core/src/platform/server/fs/path-join.ts b/packages/loot-core/src/platform/server/fs/path-join.ts index 1479912d44..12408b337b 100644 --- a/packages/loot-core/src/platform/server/fs/path-join.ts +++ b/packages/loot-core/src/platform/server/fs/path-join.ts @@ -1,2 +1,97 @@ -export declare function join(...args: string[]): string; -export type Join = typeof join; +// @ts-strict-ignore +// This code is pulled from +// https://github.com/browserify/path-browserify/blob/master/index.js#L33 + +// Resolves . and .. elements in a path with directory names +function normalizeStringPosix(path, allowAboveRoot) { + let res = ''; + let lastSegmentLength = 0; + let lastSlash = -1; + let dots = 0; + let code; + for (let i = 0; i <= path.length; ++i) { + if (i < path.length) code = path.charCodeAt(i); + else if (code === 47 /*/*/) break; + else code = 47 /*/*/; + if (code === 47 /*/*/) { + if (lastSlash === i - 1 || dots === 1) { + // NOOP + } else if (lastSlash !== i - 1 && dots === 2) { + if ( + res.length < 2 || + lastSegmentLength !== 2 || + res.charCodeAt(res.length - 1) !== 46 /*.*/ || + res.charCodeAt(res.length - 2) !== 46 /*.*/ + ) { + if (res.length > 2) { + const lastSlashIndex = res.lastIndexOf('/'); + if (lastSlashIndex !== res.length - 1) { + if (lastSlashIndex === -1) { + res = ''; + lastSegmentLength = 0; + } else { + res = res.slice(0, lastSlashIndex); + lastSegmentLength = res.length - 1 - res.lastIndexOf('/'); + } + lastSlash = i; + dots = 0; + continue; + } + } else if (res.length === 2 || res.length === 1) { + res = ''; + lastSegmentLength = 0; + lastSlash = i; + dots = 0; + continue; + } + } + if (allowAboveRoot) { + if (res.length > 0) res += '/..'; + else res = '..'; + lastSegmentLength = 2; + } + } else { + if (res.length > 0) res += '/' + path.slice(lastSlash + 1, i); + else res = path.slice(lastSlash + 1, i); + lastSegmentLength = i - lastSlash - 1; + } + lastSlash = i; + dots = 0; + } else if (code === 46 /*.*/ && dots !== -1) { + ++dots; + } else { + dots = -1; + } + } + return res; +} + +function normalizePath(path) { + if (path.length === 0) return '.'; + + const isAbsolute = path.charCodeAt(0) === 47; /*/*/ + const trailingSeparator = path.charCodeAt(path.length - 1) === 47; /*/*/ + + // Normalize the path + path = normalizeStringPosix(path, !isAbsolute); + + if (path.length === 0 && !isAbsolute) path = '.'; + if (path.length > 0 && trailingSeparator) path += '/'; + + if (isAbsolute) return '/' + path; + return path; +} + +export const join = (...args: string[]) => { + if (args.length === 0) return '.'; + let joined; + for (let i = 0; i < args.length; ++i) { + const arg = args[i]; + if (arg.length > 0) { + if (joined === undefined) joined = arg; + else joined += '/' + arg; + } + } + if (joined === undefined) return '.'; + return normalizePath(joined); +}; diff --git a/packages/loot-core/src/platform/server/fs/path-join.web.ts b/packages/loot-core/src/platform/server/fs/path-join.web.ts deleted file mode 100644 index 3486be7e98..0000000000 --- a/packages/loot-core/src/platform/server/fs/path-join.web.ts +++ /dev/null @@ -1,99 +0,0 @@ -// @ts-strict-ignore -import type * as T from './path-join'; - -// This code is pulled from -// https://github.com/browserify/path-browserify/blob/master/index.js#L33 - -// Resolves . and .. elements in a path with directory names -function normalizeStringPosix(path, allowAboveRoot) { - let res = ''; - let lastSegmentLength = 0; - let lastSlash = -1; - let dots = 0; - let code; - for (let i = 0; i <= path.length; ++i) { - if (i < path.length) code = path.charCodeAt(i); - else if (code === 47 /*/*/) break; - else code = 47 /*/*/; - if (code === 47 /*/*/) { - if (lastSlash === i - 1 || dots === 1) { - // NOOP - } else if (lastSlash !== i - 1 && dots === 2) { - if ( - res.length < 2 || - lastSegmentLength !== 2 || - res.charCodeAt(res.length - 1) !== 46 /*.*/ || - res.charCodeAt(res.length - 2) !== 46 /*.*/ - ) { - if (res.length > 2) { - const lastSlashIndex = res.lastIndexOf('/'); - if (lastSlashIndex !== res.length - 1) { - if (lastSlashIndex === -1) { - res = ''; - lastSegmentLength = 0; - } else { - res = res.slice(0, lastSlashIndex); - lastSegmentLength = res.length - 1 - res.lastIndexOf('/'); - } - lastSlash = i; - dots = 0; - continue; - } - } else if (res.length === 2 || res.length === 1) { - res = ''; - lastSegmentLength = 0; - lastSlash = i; - dots = 0; - continue; - } - } - if (allowAboveRoot) { - if (res.length > 0) res += '/..'; - else res = '..'; - lastSegmentLength = 2; - } - } else { - if (res.length > 0) res += '/' + path.slice(lastSlash + 1, i); - else res = path.slice(lastSlash + 1, i); - lastSegmentLength = i - lastSlash - 1; - } - lastSlash = i; - dots = 0; - } else if (code === 46 /*.*/ && dots !== -1) { - ++dots; - } else { - dots = -1; - } - } - return res; -} - -function normalizePath(path) { - if (path.length === 0) return '.'; - - const isAbsolute = path.charCodeAt(0) === 47; /*/*/ - const trailingSeparator = path.charCodeAt(path.length - 1) === 47; /*/*/ - - // Normalize the path - path = normalizeStringPosix(path, !isAbsolute); - - if (path.length === 0 && !isAbsolute) path = '.'; - if (path.length > 0 && trailingSeparator) path += '/'; - - if (isAbsolute) return '/' + path; - return path; -} - -export const join: T.Join = (...args) => { - if (args.length === 0) return '.'; - let joined; - for (let i = 0; i < args.length; ++i) { - const arg = args[i]; - if (arg.length > 0) { - if (joined === undefined) joined = arg; - else joined += '/' + arg; - } - } - if (joined === undefined) return '.'; - return normalizePath(joined); -}; diff --git a/packages/loot-core/src/platform/server/fs/shared.ts b/packages/loot-core/src/platform/server/fs/shared.ts index 0d57878801..4ed9363348 100644 --- a/packages/loot-core/src/platform/server/fs/shared.ts +++ b/packages/loot-core/src/platform/server/fs/shared.ts @@ -1,10 +1,8 @@ // @ts-strict-ignore import { join } from './path-join'; -import type * as T from '.'; - let documentDir; -export const _setDocumentDir: T._SetDocumentDir = dir => (documentDir = dir); +export const _setDocumentDir = dir => (documentDir = dir); export const getDocumentDir = () => { if (!documentDir) { diff --git a/packages/loot-core/src/platform/server/sqlite/index.api.ts b/packages/loot-core/src/platform/server/sqlite/index.api.ts new file mode 100644 index 0000000000..1498cc1767 --- /dev/null +++ b/packages/loot-core/src/platform/server/sqlite/index.api.ts @@ -0,0 +1,2 @@ +// oxlint-disable-next-line no-restricted-imports +export * from './index.electron'; diff --git a/packages/loot-core/src/platform/server/sqlite/index.web.test.ts b/packages/loot-core/src/platform/server/sqlite/index.test.ts similarity index 96% rename from packages/loot-core/src/platform/server/sqlite/index.web.test.ts rename to packages/loot-core/src/platform/server/sqlite/index.test.ts index 85831605fd..d07400183c 100644 --- a/packages/loot-core/src/platform/server/sqlite/index.web.test.ts +++ b/packages/loot-core/src/platform/server/sqlite/index.test.ts @@ -1,14 +1,7 @@ // @ts-strict-ignore import { patchFetchForSqlJS } from '../../../mocks/util'; -// oxlint-disable-next-line eslint/no-restricted-imports -import { - execQuery, - init, - openDatabase, - runQuery, - transaction, -} from './index.web'; +import { execQuery, init, openDatabase, runQuery, transaction } from './index'; beforeAll(async () => { const baseURL = `${__dirname}/../../../../../../node_modules/@jlongster/sql.js/dist/`; diff --git a/packages/loot-core/src/platform/server/sqlite/index.ts b/packages/loot-core/src/platform/server/sqlite/index.ts index 50a1ef6944..e2cc642290 100644 --- a/packages/loot-core/src/platform/server/sqlite/index.ts +++ b/packages/loot-core/src/platform/server/sqlite/index.ts @@ -1,5 +1,11 @@ -import type { Database, SqlJsStatic } from '@jlongster/sql.js'; -/// +// @ts-strict-ignore +import initSqlJS from '@jlongster/sql.js'; +import type { Database, SqlJsStatic, Statement } from '@jlongster/sql.js'; + +import { logger } from '../log'; + +import { normalise } from './normalise'; +import { unicodeLike } from './unicodeLike'; // Types exported from sql.js (and Emscripten) are incomplete, so we need to redefine them here type FSStream = (typeof FS)['FSStream'] & { @@ -23,38 +29,213 @@ export type SqlJsModule = SqlJsStatic & { register_for_idb: (idb: IDBDatabase) => void; }; -export declare function init(): Promise; +let SQL: SqlJsModule | null = null; -export declare function _getModule(): SqlJsModule; +export async function init({ + baseURL = process.env.PUBLIC_URL, +}: { baseURL?: string } = {}) { + // `initSqlJS` doesn't actually return a real promise, so make sure + // we're returning a real one for correct semantics + return new Promise((resolve, reject) => { + initSqlJS({ + locateFile: file => baseURL + file, + }).then( + sql => { + SQL = sql as SqlJsModule; + resolve(undefined); + }, + err => { + reject(err); + }, + ); + }); +} -export declare function prepare(db: Database, sql: string): string; +export function _getModule() { + if (SQL == null) { + throw new Error('_getModule: sql.js must be initialized first'); + } + return SQL; +} -export declare function runQuery( +function verifyParamTypes( + sql: string | Statement, + arr: (string | number)[] = [], +) { + arr.forEach(val => { + if (typeof val !== 'string' && typeof val !== 'number' && val !== null) { + throw new Error('Invalid field type ' + val + ' for sql ' + sql); + } + }); +} + +export function prepare(db: Database, sql: string) { + return db.prepare(sql); +} + +export function runQuery( db: Database, - sql: string, + sql: string | Statement, params?: (string | number)[], fetchAll?: false, ): { changes: unknown }; -export declare function runQuery( +export function runQuery( db: Database, - sql: string, + sql: string | Statement, params: (string | number)[], fetchAll: true, ): T[]; - -export declare function execQuery(db: Database, sql: string): void; - -export declare function transaction(db: Database, fn: () => void): void; - -export declare function asyncTransaction( +export function runQuery( db: Database, - fn: () => Promise, -): Promise; + sql: string | Statement, + params: (string | number)[] = [], + fetchAll = false, +): T[] | { changes: unknown } { + if (params) { + verifyParamTypes(sql, params); + } -export declare function openDatabase( - pathOrBuffer?: string | Buffer, -): Promise; + const stmt = typeof sql === 'string' ? db.prepare(sql) : sql; -export declare function closeDatabase(db: Database): void; + if (fetchAll) { + try { + stmt.bind(params); + const rows = []; -export declare function exportDatabase(db: Database): Promise; + while (stmt.step()) { + rows.push(stmt.getAsObject()); + } + + if (typeof sql === 'string') { + stmt.free(); + } else { + stmt.reset(); + } + return rows; + } catch (e) { + logger.log(sql); + throw e; + } + } else { + stmt.run(params); + return { changes: db.getRowsModified() }; + } +} + +export function execQuery(db: Database, sql: string) { + db.exec(sql); +} + +let transactionDepth = 0; + +export function transaction(db: Database, fn: () => void) { + let before, after, undo; + if (transactionDepth > 0) { + before = 'SAVEPOINT __actual_sp'; + after = 'RELEASE __actual_sp'; + undo = 'ROLLBACK TO __actual_sp'; + } else { + before = 'BEGIN'; + after = 'COMMIT'; + undo = 'ROLLBACK'; + } + + execQuery(db, before); + transactionDepth++; + + try { + fn(); + execQuery(db, after); + } catch (ex) { + execQuery(db, undo); + + if (undo !== 'ROLLBACK') { + execQuery(db, after); + } + + throw ex; + } finally { + transactionDepth--; + } +} + +// See the comment about this function in index.electron.js. You +// shouldn't normally use this. I'd like to get rid of it. +export async function asyncTransaction(db: Database, fn: () => Promise) { + // Support nested transactions by "coalescing" them into the parent + // one if one is already started + if (transactionDepth === 0) { + db.exec('BEGIN TRANSACTION'); + } + transactionDepth++; + + try { + await fn(); + } finally { + transactionDepth--; + // We always commit because rollback is more dangerous - any + // queries that ran *in-between* this async function would be + // lost. Right now we are only using transactions for speed + // purposes unfortunately + if (transactionDepth === 0) { + db.exec('COMMIT'); + } + } +} + +function regexp(regex: string, text: string) { + return new RegExp(regex).test(text || '') ? 1 : 0; +} + +export async function openDatabase(pathOrBuffer?: string | Uint8Array) { + let db = null; + if (pathOrBuffer) { + if (typeof pathOrBuffer !== 'string') { + db = new SQL.Database(pathOrBuffer); + } else { + const path = pathOrBuffer; + if (path !== ':memory:') { + if (typeof SharedArrayBuffer === 'undefined') { + const stream = SQL.FS.open(SQL.FS.readlink(path), 'a+'); + await stream.node.contents.readIfFallback(); + SQL.FS.close(stream); + } + + db = new SQL.Database( + path.includes('/blocked') ? path : SQL.FS.readlink(path), + // @ts-expect-error 2nd argument missed in sql.js types + { filename: true }, + ); + db.exec(` + PRAGMA journal_mode=MEMORY; + PRAGMA cache_size=-10000; + `); + } + } + } + + if (db === null) { + db = new SQL.Database(); + } + + // Define Unicode-aware LOWER, UPPER, and LIKE implementation. + // This is necessary because sql.js uses SQLite build without ICU support. + // + // Note that this function should ideally be created with a deterministic flag + // to allow SQLite to better optimize calls to it by factoring them out of inner loops + // but SQL.js does not support this: https://github.com/sql-js/sql.js/issues/551 + db.create_function('UNICODE_LOWER', arg => arg?.toLowerCase()); + db.create_function('UNICODE_UPPER', arg => arg?.toUpperCase()); + db.create_function('UNICODE_LIKE', unicodeLike); + db.create_function('REGEXP', regexp); + db.create_function('NORMALISE', normalise); + return db; +} + +export function closeDatabase(db: Database) { + db.close(); +} + +export async function exportDatabase(db: Database) { + return db.export(); +} diff --git a/packages/loot-core/src/platform/server/sqlite/index.web.ts b/packages/loot-core/src/platform/server/sqlite/index.web.ts deleted file mode 100644 index 44d3358d8a..0000000000 --- a/packages/loot-core/src/platform/server/sqlite/index.web.ts +++ /dev/null @@ -1,218 +0,0 @@ -// @ts-strict-ignore -import initSqlJS from '@jlongster/sql.js'; -import type { Database } from '@jlongster/sql.js'; - -import { logger } from '../log'; - -import { normalise } from './normalise'; -import { unicodeLike } from './unicodeLike'; - -import type { SqlJsModule } from '.'; - -let SQL: SqlJsModule | null = null; - -export async function init({ - baseURL = process.env.PUBLIC_URL, -}: { baseURL?: string } = {}) { - // `initSqlJS` doesn't actually return a real promise, so make sure - // we're returning a real one for correct semantics - return new Promise((resolve, reject) => { - initSqlJS({ - locateFile: file => baseURL + file, - }).then( - sql => { - SQL = sql as SqlJsModule; - resolve(undefined); - }, - err => { - reject(err); - }, - ); - }); -} - -export function _getModule() { - if (SQL == null) { - throw new Error('_getModule: sql.js must be initialized first'); - } - return SQL; -} - -function verifyParamTypes(sql: string, arr: (string | number)[] = []) { - arr.forEach(val => { - if (typeof val !== 'string' && typeof val !== 'number' && val !== null) { - throw new Error('Invalid field type ' + val + ' for sql ' + sql); - } - }); -} - -export function prepare(db: Database, sql: string) { - return db.prepare(sql); -} - -export function runQuery( - db: Database, - sql: string, - params?: (string | number)[], - fetchAll?: false, -): { changes: unknown }; -export function runQuery( - db: Database, - sql: string, - params: (string | number)[], - fetchAll: true, -): unknown[]; -export function runQuery( - db: Database, - sql: string, - params: (string | number)[] = [], - fetchAll = false, -): unknown[] | { changes: unknown } { - if (params) { - verifyParamTypes(sql, params); - } - - const stmt = typeof sql === 'string' ? db.prepare(sql) : sql; - - if (fetchAll) { - try { - stmt.bind(params); - const rows = []; - - while (stmt.step()) { - rows.push(stmt.getAsObject()); - } - - if (typeof sql === 'string') { - stmt.free(); - } else { - stmt.reset(); - } - return rows; - } catch (e) { - logger.log(sql); - throw e; - } - } else { - stmt.run(params); - return { changes: db.getRowsModified() }; - } -} - -export function execQuery(db: Database, sql: string) { - db.exec(sql); -} - -let transactionDepth = 0; - -export function transaction(db: Database, fn: () => void) { - let before, after, undo; - if (transactionDepth > 0) { - before = 'SAVEPOINT __actual_sp'; - after = 'RELEASE __actual_sp'; - undo = 'ROLLBACK TO __actual_sp'; - } else { - before = 'BEGIN'; - after = 'COMMIT'; - undo = 'ROLLBACK'; - } - - execQuery(db, before); - transactionDepth++; - - try { - fn(); - execQuery(db, after); - } catch (ex) { - execQuery(db, undo); - - if (undo !== 'ROLLBACK') { - execQuery(db, after); - } - - throw ex; - } finally { - transactionDepth--; - } -} - -// See the comment about this function in index.electron.js. You -// shouldn't normally use this. I'd like to get rid of it. -export async function asyncTransaction(db: Database, fn: () => Promise) { - // Support nested transactions by "coalescing" them into the parent - // one if one is already started - if (transactionDepth === 0) { - db.exec('BEGIN TRANSACTION'); - } - transactionDepth++; - - try { - await fn(); - } finally { - transactionDepth--; - // We always commit because rollback is more dangerous - any - // queries that ran *in-between* this async function would be - // lost. Right now we are only using transactions for speed - // purposes unfortunately - if (transactionDepth === 0) { - db.exec('COMMIT'); - } - } -} - -function regexp(regex: string, text: string) { - return new RegExp(regex).test(text || '') ? 1 : 0; -} - -export async function openDatabase(pathOrBuffer?: string | Buffer) { - let db = null; - if (pathOrBuffer) { - if (typeof pathOrBuffer !== 'string') { - db = new SQL.Database(pathOrBuffer); - } else { - const path = pathOrBuffer; - if (path !== ':memory:') { - if (typeof SharedArrayBuffer === 'undefined') { - const stream = SQL.FS.open(SQL.FS.readlink(path), 'a+'); - await stream.node.contents.readIfFallback(); - SQL.FS.close(stream); - } - - db = new SQL.Database( - path.includes('/blocked') ? path : SQL.FS.readlink(path), - // @ts-expect-error 2nd argument missed in sql.js types - { filename: true }, - ); - db.exec(` - PRAGMA journal_mode=MEMORY; - PRAGMA cache_size=-10000; - `); - } - } - } - - if (db === null) { - db = new SQL.Database(); - } - - // Define Unicode-aware LOWER, UPPER, and LIKE implementation. - // This is necessary because sql.js uses SQLite build without ICU support. - // - // Note that this function should ideally be created with a deterministic flag - // to allow SQLite to better optimize calls to it by factoring them out of inner loops - // but SQL.js does not support this: https://github.com/sql-js/sql.js/issues/551 - db.create_function('UNICODE_LOWER', arg => arg?.toLowerCase()); - db.create_function('UNICODE_UPPER', arg => arg?.toUpperCase()); - db.create_function('UNICODE_LIKE', unicodeLike); - db.create_function('REGEXP', regexp); - db.create_function('NORMALISE', normalise); - return db; -} - -export function closeDatabase(db: Database) { - db.close(); -} - -export async function exportDatabase(db: Database) { - return db.export(); -} diff --git a/packages/loot-core/src/server/db/index.ts b/packages/loot-core/src/server/db/index.ts index a4f4028e12..1bfabb4da3 100644 --- a/packages/loot-core/src/server/db/index.ts +++ b/packages/loot-core/src/server/db/index.ts @@ -7,7 +7,7 @@ import { setClock, Timestamp, } from '@actual-app/crdt'; -import type { Database } from '@jlongster/sql.js'; +import type { Database, Statement } from '@jlongster/sql.js'; import { LRUCache } from 'lru-cache'; import { v4 as uuidv4 } from 'uuid'; @@ -110,19 +110,19 @@ export async function loadClock() { // Functions export function runQuery( - sql: string, + sql: string | Statement, params?: Array, fetchAll?: false, ): { changes: unknown }; export function runQuery( - sql: string, + sql: string | Statement, params: Array | undefined, fetchAll: true, ): T[]; export function runQuery( - sql: string, + sql: string | Statement, params: (string | number)[], fetchAll: boolean, ) { @@ -139,7 +139,7 @@ export function execQuery(sql: string) { // This manages an LRU cache of prepared query statements. This is // only needed in hot spots when you are running lots of queries. -let _queryCache = new LRUCache({ max: 100 }); +let _queryCache = new LRUCache({ max: 100 }); export function cache(sql: string) { const cached = _queryCache.get(sql); if (cached) { @@ -152,7 +152,7 @@ export function cache(sql: string) { } function resetQueryCache() { - _queryCache = new LRUCache({ max: 100 }); + _queryCache = new LRUCache({ max: 100 }); } export function transaction(fn: () => void) { diff --git a/packages/loot-core/src/server/encryption/encryption-internals.api.ts b/packages/loot-core/src/server/encryption/encryption-internals.api.ts new file mode 100644 index 0000000000..3e0af5dfa7 --- /dev/null +++ b/packages/loot-core/src/server/encryption/encryption-internals.api.ts @@ -0,0 +1,2 @@ +// oxlint-disable-next-line no-restricted-imports +export * from './encryption-internals.electron'; diff --git a/packages/loot-core/src/server/encryption/encryption-internals.electron.ts b/packages/loot-core/src/server/encryption/encryption-internals.electron.ts new file mode 100644 index 0000000000..7667aa7db4 --- /dev/null +++ b/packages/loot-core/src/server/encryption/encryption-internals.electron.ts @@ -0,0 +1,89 @@ +// @ts-strict-ignore +import crypto from 'crypto'; + +import type * as T from './encryption-internals'; + +const ENCRYPTION_ALGORITHM = 'aes-256-gcm' as const; + +export const randomBytes: typeof T.randomBytes = n => { + return crypto.randomBytes(n); +}; + +export const encrypt: typeof T.encrypt = async (masterKey, value) => { + const masterKeyBuffer = masterKey.getValue().raw; + // let iv = createKeyBuffer({ numBytes: 12, secret: masterKeyBuffer }); + const iv = crypto.randomBytes(12); + const cipher = crypto.createCipheriv( + ENCRYPTION_ALGORITHM, + masterKeyBuffer, + iv, + ); + let encrypted = cipher.update(value); + encrypted = Buffer.concat([encrypted, cipher.final()]); + + const authTag = cipher.getAuthTag(); + + return { + value: encrypted, + meta: { + keyId: masterKey.getId(), + algorithm: ENCRYPTION_ALGORITHM, + iv: iv.toString('base64'), + authTag: authTag.toString('base64'), + }, + }; +}; + +export const decrypt: typeof T.decrypt = async (masterKey, encrypted, meta) => { + const masterKeyBuffer = masterKey.getValue().raw; + const { algorithm, iv: originalIv, authTag: originalAuthTag } = meta; + const iv = Buffer.from(originalIv, 'base64'); + const authTag = Buffer.from(originalAuthTag, 'base64'); + + const decipher = crypto.createDecipheriv(algorithm, masterKeyBuffer, iv); + decipher.setAuthTag(authTag); + + let decrypted = decipher.update(encrypted); + decrypted = Buffer.concat([decrypted, decipher.final()]); + return decrypted; +}; + +// @ts-expect-error fix me +export const createKey: typeof T.createKey = async ({ secret, salt }) => { + const buffer = createKeyBuffer({ secret, salt }); + return { + raw: buffer, + base64: buffer.toString('base64'), + }; +}; + +// @ts-expect-error fix me +export const importKey: typeof T.importKey = str => { + return { + raw: Buffer.from(str, 'base64'), + base64: str, + }; +}; + +/** + * Generates a Buffer of a desired byte length to be used as either an encryption key or an initialization vector. + * + * @private + */ +function createKeyBuffer({ + numBytes, + secret, + salt, +}: { + numBytes?: number; + secret?: string; + salt?: string; +}) { + return crypto.pbkdf2Sync( + secret || crypto.randomBytes(128).toString('base64'), + salt || crypto.randomBytes(32).toString('base64'), + 10000, + numBytes || 32, + 'sha512', + ); +} diff --git a/packages/loot-core/src/server/encryption/encryption-internals.ts b/packages/loot-core/src/server/encryption/encryption-internals.ts index 9384da4e3b..dc35b5b892 100644 --- a/packages/loot-core/src/server/encryption/encryption-internals.ts +++ b/packages/loot-core/src/server/encryption/encryption-internals.ts @@ -1,85 +1,109 @@ // @ts-strict-ignore -import crypto from 'crypto'; +const ENCRYPTION_ALGORITHM = 'aes-256-gcm'; -const ENCRYPTION_ALGORITHM = 'aes-256-gcm' as const; +function browserAlgorithmName(name) { + switch (name) { + case 'aes-256-gcm': + return 'AES-GCM'; + default: + throw new Error('unsupported crypto algorithm: ' + name); + } +} export function randomBytes(n) { - return crypto.randomBytes(n); + return Buffer.from(crypto.getRandomValues(new Uint8Array(n))); } export async function encrypt(masterKey, value) { - const masterKeyBuffer = masterKey.getValue().raw; - // let iv = createKeyBuffer({ numBytes: 12, secret: masterKeyBuffer }); - const iv = crypto.randomBytes(12); - const cipher = crypto.createCipheriv( - ENCRYPTION_ALGORITHM, - masterKeyBuffer, - iv, - ); - let encrypted = cipher.update(value); - encrypted = Buffer.concat([encrypted, cipher.final()]); + const iv = crypto.getRandomValues(new Uint8Array(12)); - const authTag = cipher.getAuthTag(); + const encryptedArrayBuffer = await crypto.subtle.encrypt( + { + name: browserAlgorithmName(ENCRYPTION_ALGORITHM), + iv, + tagLength: 128, + }, + masterKey.getValue().raw, + value, + ); + + const encrypted = Buffer.from(encryptedArrayBuffer); + + // Strip the auth tag off the end + const authTag = encrypted.slice(-16); + const strippedEncrypted = encrypted.slice(0, -16); return { - value: encrypted, + value: strippedEncrypted, meta: { keyId: masterKey.getId(), algorithm: ENCRYPTION_ALGORITHM, - iv: iv.toString('base64'), + iv: Buffer.from(iv).toString('base64'), authTag: authTag.toString('base64'), }, }; } export async function decrypt(masterKey, encrypted, meta) { - const masterKeyBuffer = masterKey.getValue().raw; - const { algorithm, iv: originalIv, authTag: originalAuthTag } = meta; - const iv = Buffer.from(originalIv, 'base64'); - const authTag = Buffer.from(originalAuthTag, 'base64'); + const { algorithm, iv, authTag } = meta; - const decipher = crypto.createDecipheriv(algorithm, masterKeyBuffer, iv); - decipher.setAuthTag(authTag); + const decrypted = await crypto.subtle.decrypt( + { + name: browserAlgorithmName(algorithm), + iv: Buffer.from(iv, 'base64'), + tagLength: 128, + }, + masterKey.getValue().raw, + Buffer.concat([encrypted, Buffer.from(authTag, 'base64')]), + ); - let decrypted = decipher.update(encrypted); - decrypted = Buffer.concat([decrypted, decipher.final()]); - return decrypted; + return Buffer.from(decrypted); } export async function createKey({ secret, salt }) { - const buffer = createKeyBuffer({ secret, salt }); + const passwordBuffer = Buffer.from(secret); + const saltBuffer = Buffer.from(salt); + + const passwordKey = await crypto.subtle.importKey( + 'raw', + passwordBuffer, + { name: 'PBKDF2' }, + false, + ['deriveBits', 'deriveKey'], + ); + + const derivedKey = await crypto.subtle.deriveKey( + { + name: 'PBKDF2', + hash: 'SHA-512', + salt: saltBuffer, + iterations: 10000, + }, + passwordKey, + { name: 'AES-GCM', length: 256 }, + true, + ['encrypt', 'decrypt'], + ); + + const exported = await crypto.subtle.exportKey('raw', derivedKey); + return { - raw: buffer, - base64: buffer.toString('base64'), + raw: derivedKey, + base64: Buffer.from(exported).toString('base64'), }; } export async function importKey(str) { + const key = await crypto.subtle.importKey( + 'raw', + Buffer.from(str, 'base64'), + { name: 'AES-GCM' }, + false, + ['encrypt', 'decrypt'], + ); + return { - raw: Buffer.from(str, 'base64'), + raw: key, base64: str, }; } - -/** - * Generates a Buffer of a desired byte length to be used as either an encryption key or an initialization vector. - * - * @private - */ -function createKeyBuffer({ - numBytes, - secret, - salt, -}: { - numBytes?: number; - secret?: string; - salt?: string; -}) { - return crypto.pbkdf2Sync( - secret || crypto.randomBytes(128).toString('base64'), - salt || crypto.randomBytes(32).toString('base64'), - 10000, - numBytes || 32, - 'sha512', - ); -} diff --git a/packages/loot-core/src/server/encryption/encryption-internals.web.ts b/packages/loot-core/src/server/encryption/encryption-internals.web.ts deleted file mode 100644 index dc35b5b892..0000000000 --- a/packages/loot-core/src/server/encryption/encryption-internals.web.ts +++ /dev/null @@ -1,109 +0,0 @@ -// @ts-strict-ignore -const ENCRYPTION_ALGORITHM = 'aes-256-gcm'; - -function browserAlgorithmName(name) { - switch (name) { - case 'aes-256-gcm': - return 'AES-GCM'; - default: - throw new Error('unsupported crypto algorithm: ' + name); - } -} - -export function randomBytes(n) { - return Buffer.from(crypto.getRandomValues(new Uint8Array(n))); -} - -export async function encrypt(masterKey, value) { - const iv = crypto.getRandomValues(new Uint8Array(12)); - - const encryptedArrayBuffer = await crypto.subtle.encrypt( - { - name: browserAlgorithmName(ENCRYPTION_ALGORITHM), - iv, - tagLength: 128, - }, - masterKey.getValue().raw, - value, - ); - - const encrypted = Buffer.from(encryptedArrayBuffer); - - // Strip the auth tag off the end - const authTag = encrypted.slice(-16); - const strippedEncrypted = encrypted.slice(0, -16); - - return { - value: strippedEncrypted, - meta: { - keyId: masterKey.getId(), - algorithm: ENCRYPTION_ALGORITHM, - iv: Buffer.from(iv).toString('base64'), - authTag: authTag.toString('base64'), - }, - }; -} - -export async function decrypt(masterKey, encrypted, meta) { - const { algorithm, iv, authTag } = meta; - - const decrypted = await crypto.subtle.decrypt( - { - name: browserAlgorithmName(algorithm), - iv: Buffer.from(iv, 'base64'), - tagLength: 128, - }, - masterKey.getValue().raw, - Buffer.concat([encrypted, Buffer.from(authTag, 'base64')]), - ); - - return Buffer.from(decrypted); -} - -export async function createKey({ secret, salt }) { - const passwordBuffer = Buffer.from(secret); - const saltBuffer = Buffer.from(salt); - - const passwordKey = await crypto.subtle.importKey( - 'raw', - passwordBuffer, - { name: 'PBKDF2' }, - false, - ['deriveBits', 'deriveKey'], - ); - - const derivedKey = await crypto.subtle.deriveKey( - { - name: 'PBKDF2', - hash: 'SHA-512', - salt: saltBuffer, - iterations: 10000, - }, - passwordKey, - { name: 'AES-GCM', length: 256 }, - true, - ['encrypt', 'decrypt'], - ); - - const exported = await crypto.subtle.exportKey('raw', derivedKey); - - return { - raw: derivedKey, - base64: Buffer.from(exported).toString('base64'), - }; -} - -export async function importKey(str) { - const key = await crypto.subtle.importKey( - 'raw', - Buffer.from(str, 'base64'), - { name: 'AES-GCM' }, - false, - ['encrypt', 'decrypt'], - ); - - return { - raw: key, - base64: str, - }; -} diff --git a/packages/loot-core/src/shared/__mocks__/platform.web.ts b/packages/loot-core/src/shared/__mocks__/platform.ts similarity index 100% rename from packages/loot-core/src/shared/__mocks__/platform.web.ts rename to packages/loot-core/src/shared/__mocks__/platform.ts diff --git a/packages/loot-core/src/shared/platform.electron.ts b/packages/loot-core/src/shared/platform.electron.ts index d24038afa4..0d6b8b2723 100644 --- a/packages/loot-core/src/shared/platform.electron.ts +++ b/packages/loot-core/src/shared/platform.electron.ts @@ -1,19 +1,21 @@ import os from 'os'; +import type * as T from './platform'; + const isWindows = os.platform() === 'win32'; const isMac = os.platform() === 'darwin'; const isLinux = os.platform() === 'linux'; export const isPlaywright = false; -export const OS: 'windows' | 'mac' | 'linux' | 'unknown' = isWindows +export const OS: typeof T.OS = isWindows ? 'windows' : isMac ? 'mac' : isLinux ? 'linux' : 'unknown'; -export const env: 'web' | 'mobile' | 'unknown' = 'unknown'; -export const isBrowser = false; +export const env: typeof T.env = 'unknown'; +export const isBrowser: typeof T.isBrowser = false; -export const isIOSAgent = false; +export const isIOSAgent: typeof T.isIOSAgent = false; diff --git a/packages/loot-core/src/shared/platform.ts b/packages/loot-core/src/shared/platform.ts index 5417b58c65..f4bc5f57f6 100644 --- a/packages/loot-core/src/shared/platform.ts +++ b/packages/loot-core/src/shared/platform.ts @@ -1,7 +1,20 @@ -export const isPlaywright = false; +import { UAParser } from 'ua-parser-js'; -export const OS: 'windows' | 'mac' | 'linux' | 'unknown' = 'unknown'; -export const env: 'web' | 'mobile' | 'unknown' = 'unknown'; -export const isBrowser = false; +const isWindows = + navigator.platform && navigator.platform.toLowerCase() === 'win32'; -export const isIOSAgent = false; +const isMac = + navigator.platform && navigator.platform.toUpperCase().indexOf('MAC') >= 0; + +export const isPlaywright = navigator.userAgent === 'playwright'; + +export const OS: 'windows' | 'mac' | 'linux' | 'unknown' = isWindows + ? 'windows' + : isMac + ? 'mac' + : 'linux'; +export const env: 'web' | 'mobile' | 'unknown' = 'web'; +export const isBrowser: boolean = true; + +const agent = UAParser(navigator.userAgent); +export const isIOSAgent = agent.browser.name === 'Mobile Safari'; diff --git a/packages/loot-core/src/shared/platform.web.ts b/packages/loot-core/src/shared/platform.web.ts deleted file mode 100644 index dfae65bfe5..0000000000 --- a/packages/loot-core/src/shared/platform.web.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { UAParser } from 'ua-parser-js'; - -const isWindows = - navigator.platform && navigator.platform.toLowerCase() === 'win32'; - -const isMac = - navigator.platform && navigator.platform.toUpperCase().indexOf('MAC') >= 0; - -export const isPlaywright = navigator.userAgent === 'playwright'; - -export const OS: 'windows' | 'mac' | 'linux' | 'unknown' = isWindows - ? 'windows' - : isMac - ? 'mac' - : 'linux'; -export const env: 'web' | 'mobile' | 'unknown' = 'web'; -export const isBrowser = true; - -const agent = UAParser(navigator.userAgent); -export const isIOSAgent = agent.browser.name === 'Mobile Safari'; diff --git a/packages/loot-core/vite.api.config.ts b/packages/loot-core/vite.api.config.ts index e302ad2c05..432c269a8d 100644 --- a/packages/loot-core/vite.api.config.ts +++ b/packages/loot-core/vite.api.config.ts @@ -34,9 +34,6 @@ export default defineConfig(({ mode }) => { '.api.js', '.api.ts', '.api.tsx', - '.electron.js', - '.electron.ts', - '.electron.tsx', '.js', '.ts', '.tsx', diff --git a/packages/loot-core/vite.config.ts b/packages/loot-core/vite.config.ts index 237c163594..4a3e2072f5 100644 --- a/packages/loot-core/vite.config.ts +++ b/packages/loot-core/vite.config.ts @@ -61,15 +61,7 @@ export default defineConfig(({ mode }) => { }, }, resolve: { - extensions: [ - '.web.js', - '.web.ts', - '.web.tsx', - '.js', - '.ts', - '.tsx', - '.json', - ], + extensions: ['.js', '.ts', '.tsx', '.json'], alias: [ { find: /^@actual-app\/crdt(\/.*)?$/, diff --git a/packages/loot-core/vitest.config.ts b/packages/loot-core/vitest.config.ts index d7287868a6..1eb3ea8d0a 100644 --- a/packages/loot-core/vitest.config.ts +++ b/packages/loot-core/vitest.config.ts @@ -19,7 +19,11 @@ export default defineConfig({ test: { globals: true, setupFiles: ['./src/mocks/setup.ts'], - exclude: ['src/**/*.web.test.(js|jsx|ts|tsx)', 'node_modules'], + exclude: [ + 'src/platform/server/sqlite/index.test.ts', + 'src/platform/server/fs/index.test.ts', + 'node_modules', + ], onConsoleLog(log: string, type: 'stdout' | 'stderr'): boolean | void { // print only console.error return type === 'stderr'; diff --git a/packages/loot-core/vitest.web.config.ts b/packages/loot-core/vitest.web.config.ts index e985019ddf..2ec14a10cb 100644 --- a/packages/loot-core/vitest.web.config.ts +++ b/packages/loot-core/vitest.web.config.ts @@ -5,7 +5,6 @@ import { defineConfig } from 'vitest/config'; const resolveExtensions = [ '.testing.ts', - '.web.ts', '.mjs', '.js', '.mts', @@ -20,7 +19,10 @@ export default defineConfig({ test: { environment: 'jsdom', globals: true, - include: ['src/**/*.web.test.(js|jsx|ts|tsx)'], + include: [ + 'src/platform/server/sqlite/index.test.ts', + 'src/platform/server/fs/index.test.ts', + ], maxWorkers: 2, }, resolve: { diff --git a/tsconfig.json b/tsconfig.json index 581705df21..966fcd78ef 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -54,6 +54,7 @@ "**/client-build/*", "**/dist/*", "**/lib-dist/*", + "packages/api/@types", "**/test-results/*", "**/playwright-report/*", "**/service-worker/*", diff --git a/upcoming-release-notes/7033.md b/upcoming-release-notes/7033.md new file mode 100644 index 0000000000..47011bdd55 --- /dev/null +++ b/upcoming-release-notes/7033.md @@ -0,0 +1,6 @@ +--- +category: Maintenance +authors: [MatissJanis] +--- + +Remove usage of 'web' file types