diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index ef5f163717..55e9dcee78 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -46,13 +46,12 @@ jobs: # via ConfigurationPage.createTestFile()) is still rendered in a # production build. Without it, e2e tests would time out waiting for # a button that was tree-shaken out. + # --skip-translations keeps VRT screenshots deterministic by rendering + # source-code English instead of upstream Weblate en.json (which can + # drift between snapshot capture and test runs). env: REACT_APP_NETLIFY: 'true' - run: | - yarn workspace plugins-service build - yarn workspace @actual-app/crdt build - yarn workspace @actual-app/core build:browser - yarn workspace @actual-app/web build:browser + run: yarn build:browser --skip-translations - name: Upload build artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: diff --git a/.gitignore b/.gitignore index f26446a5b9..83fc734300 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,9 @@ bundle.desktop.js.map bundle.mobile.js bundle.mobile.js.map +# Python virtualenv (Electron CI provisions one at the repo root for setuptools) +.venv/ + # Yarn .pnp.* .yarn/* diff --git a/bin/package-browser b/bin/package-browser index fe6c178d1a..65ed276e8a 100755 --- a/bin/package-browser +++ b/bin/package-browser @@ -4,21 +4,30 @@ ROOT=`dirname $0` cd "$ROOT/.." -echo "Updating translations..." -if ! [ -d packages/desktop-client/locale ]; then - git clone https://github.com/actualbudget/translations packages/desktop-client/locale +SKIP_TRANSLATIONS=false +while [[ $# -gt 0 ]]; do + case "$1" in + --skip-translations) + SKIP_TRANSLATIONS=true + shift + ;; + *) + echo "Unknown argument: $1" >&2 + exit 1 + ;; + esac +done + +if [ "$SKIP_TRANSLATIONS" = false ]; then + echo "Updating translations..." + if ! [ -d packages/desktop-client/locale ]; then + git clone https://github.com/actualbudget/translations packages/desktop-client/locale + fi + pushd packages/desktop-client/locale > /dev/null + git checkout . + git pull + popd > /dev/null + packages/desktop-client/bin/remove-untranslated-languages fi -pushd packages/desktop-client/locale > /dev/null -git checkout . -git pull -popd > /dev/null -packages/desktop-client/bin/remove-untranslated-languages -export NODE_OPTIONS="--max-old-space-size=4096" - -yarn workspace @actual-app/crdt build -yarn workspace plugins-service build -yarn workspace @actual-app/core build:browser -yarn workspace @actual-app/web build:browser - -echo "packages/desktop-client/build" +lage build:browser --to=@actual-app/web diff --git a/bin/package-electron b/bin/package-electron index 1525d29512..6e4fbd6d41 100755 --- a/bin/package-electron +++ b/bin/package-electron @@ -57,8 +57,7 @@ yarn workspace @actual-app/core build:node yarn workspace @actual-app/web build --mode=desktop # electron specific build # required for running the sync-server server -yarn workspace @actual-app/core build:browser -yarn workspace @actual-app/web build:browser +yarn build:browser yarn workspace @actual-app/sync-server build # Emit @actual-app/core declarations so desktop-electron (which includes typings/window.ts) can build diff --git a/lage.config.js b/lage.config.js index 40114b512a..a14ac13d38 100644 --- a/lage.config.js +++ b/lage.config.js @@ -25,6 +25,14 @@ module.exports = { outputGlob: BUILD_OUTPUT_GLOBS, }, }, + // Not cached: the script stages files into public/ and build-stats/ that + // fall outside BUILD_OUTPUT_GLOBS, so a cache hit would skip the side + // effects. + 'build:browser': { + type: 'npmScript', + dependsOn: ['^build'], + cache: false, + }, }, cacheOptions: { cacheStorageConfig: { diff --git a/package.json b/package.json index aecdf954b9..83d7920807 100644 --- a/package.json +++ b/package.json @@ -24,18 +24,16 @@ "start:server-dev": "NODE_ENV=development BROWSER_OPEN=localhost:5006 yarn npm-run-all --parallel 'start:server-monitor' 'start'", "start:desktop": "yarn desktop-dependencies && npm-run-all --parallel 'start:desktop-*'", "start:docs": "yarn workspace docs start", - "desktop-dependencies": "npm-run-all --parallel rebuild-electron build:browser-backend build:plugins-service", + "desktop-dependencies": "npm-run-all --parallel rebuild-electron build:plugins-service", "start:desktop-node": "yarn workspace @actual-app/core watch:node", "start:desktop-client": "yarn workspace @actual-app/web watch", "start:desktop-server-client": "yarn workspace @actual-app/web build:browser", "start:desktop-electron": "yarn workspace desktop-electron watch", - "start:browser": "yarn workspace plugins-service build-dev && npm-run-all --parallel 'start:browser-*'", + "start:browser": "npm-run-all --parallel 'start:browser-*' 'start:service-plugins'", "start:service-plugins": "yarn workspace plugins-service watch", - "start:browser-backend": "yarn workspace @actual-app/core watch:browser", "start:browser-frontend": "yarn workspace @actual-app/web start:browser", "start:storybook": "yarn workspace @actual-app/components start:storybook", "build": "lage build", - "build:browser-backend": "yarn workspace @actual-app/core build:browser", "build:server": "yarn build:browser && yarn workspace @actual-app/sync-server build", "build:browser": "./bin/package-browser", "build:desktop": "./bin/package-electron", diff --git a/packages/desktop-client/.gitignore b/packages/desktop-client/.gitignore index 7bc018d77e..49a135ca82 100644 --- a/packages/desktop-client/.gitignore +++ b/packages/desktop-client/.gitignore @@ -8,6 +8,7 @@ coverage test-results playwright-report blob-report +.playwright-cli # production build diff --git a/packages/desktop-client/bin/build-browser b/packages/desktop-client/bin/build-browser deleted file mode 100755 index 0b5a283cfb..0000000000 --- a/packages/desktop-client/bin/build-browser +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh -ex - -ROOT=`dirname $0` -cd "$ROOT/.." - -echo "Building the browser..." - -rm -fr build - -export REACT_APP_BACKEND_WORKER_HASH=`ls "$ROOT"/../public/kcab/kcab.worker.*.js | sed 's/.*kcab\.worker\.\(.*\)\.js/\1/'` - -yarn build --mode=browser - -rm -fr build-stats -mkdir build-stats -mv build/kcab/stats.json build-stats/loot-core-stats.json -mv ./stats.json build-stats/web-stats.json diff --git a/packages/desktop-client/package.json b/packages/desktop-client/package.json index 82ba95415e..ce4fb1d62a 100644 --- a/packages/desktop-client/package.json +++ b/packages/desktop-client/package.json @@ -104,7 +104,7 @@ "start:browser": "cross-env ./bin/watch-browser", "watch": "cross-env BROWSER=none yarn start", "build": "vite build", - "build:browser": "cross-env ./bin/build-browser", + "build:browser": "vite build --mode=browser", "generate:i18n": "i18next", "test": "vitest --run", "validate:theme-catalog": "node --experimental-strip-types bin/validate-theme-catalog.mts", @@ -164,6 +164,7 @@ "mdast-util-newline-to-break": "^2.0.0", "memoize-one": "^6.0.0", "pikaday": "1.8.2", + "plugins-service": "workspace:*", "promise-retry": "^2.0.1", "re-resizable": "^6.11.2", "react": "19.2.4", diff --git a/packages/desktop-client/tsconfig.json b/packages/desktop-client/tsconfig.json index eda13c79d2..9285906bae 100644 --- a/packages/desktop-client/tsconfig.json +++ b/packages/desktop-client/tsconfig.json @@ -24,6 +24,9 @@ }, { "path": "../loot-core" + }, + { + "path": "../plugins-service" } ], "include": ["**/*.ts", "src/**/*.tsx", "src/**/*.js"], diff --git a/packages/desktop-client/vite.config.mts b/packages/desktop-client/vite.config.mts index 3fb9477818..7e1ba31654 100644 --- a/packages/desktop-client/vite.config.mts +++ b/packages/desktop-client/vite.config.mts @@ -1,3 +1,7 @@ +import { spawn } from 'node:child_process'; +import type { ChildProcess } from 'node:child_process'; +import { createReadStream } from 'node:fs'; +import { cp, mkdir, readdir, rename, rm, writeFile } from 'node:fs/promises'; import * as path from 'path'; import { fileURLToPath } from 'url'; @@ -8,7 +12,7 @@ import react, { reactCompilerPreset } from '@vitejs/plugin-react'; import type { PreRenderedAsset } from 'rolldown'; import { visualizer } from 'rollup-plugin-visualizer'; /// -import { defineConfig, loadEnv } from 'vite'; +import { build, defineConfig, loadEnv } from 'vite'; import type { Plugin } from 'vite'; import { VitePWA } from 'vite-plugin-pwa'; @@ -103,7 +107,163 @@ const injectShims = (): Plugin[] => { // https://vitejs.dev/config/ -export default defineConfig(async ({ mode }) => { +const lootCoreRoot = path.resolve(__dirname, '../loot-core'); +const lootCoreOutDir = path.resolve(lootCoreRoot, 'lib-dist/browser'); +const lootCoreConfig = path.resolve(lootCoreRoot, 'vite.config.mts'); +const sqlWasmSrc = path.resolve( + __dirname, + '../../node_modules/@jlongster/sql.js/dist/sql-wasm.wasm', +); +const publicDir = path.resolve(__dirname, 'public'); +const publicDataDir = path.resolve(publicDir, 'data'); +const publicKcabDir = path.resolve(publicDir, 'kcab'); +const buildStatsDir = path.resolve(__dirname, 'build-stats'); +const pluginsServiceDistDir = path.resolve( + __dirname, + '../plugins-service/dist', +); +const serviceWorkerDir = path.resolve(__dirname, 'service-worker'); + +const WORKER_FILENAME_RE = /^kcab\.worker\.(.+)\.js$/; + +async function extractWorkerHash(): Promise { + const files = await readdir(lootCoreOutDir); + for (const f of files) { + const match = f.match(WORKER_FILENAME_RE); + if (match) return match[1]; + } + throw new Error( + `loot-core worker build produced no hashed output at ${lootCoreOutDir}`, + ); +} + +// Serve loot-core worker assets with correct content types so the browser can +// stream-compile the sql.js wasm module. +const CONTENT_TYPES: Record = { + '.js': 'application/javascript', + '.mjs': 'application/javascript', + '.map': 'application/json', + '.wasm': 'application/wasm', +}; + +async function stagePluginsService(): Promise { + await rm(serviceWorkerDir, { recursive: true, force: true }); + await cp(pluginsServiceDistDir, serviceWorkerDir, { recursive: true }); +} + +async function stagePublicData(): Promise { + const migrationsDest = path.resolve(publicDataDir, 'migrations'); + await mkdir(publicDataDir, { recursive: true }); + await rm(migrationsDest, { recursive: true, force: true }); + await Promise.all([ + cp(path.resolve(lootCoreRoot, 'migrations'), migrationsDest, { + recursive: true, + }), + cp( + path.resolve(lootCoreRoot, 'default-db.sqlite'), + path.resolve(publicDataDir, 'default-db.sqlite'), + ), + cp(sqlWasmSrc, path.resolve(publicDir, 'sql-wasm.wasm')), + ]); + + const entries = await readdir(publicDataDir, { + recursive: true, + withFileTypes: true, + }); + const files = entries + .filter(e => e.isFile()) + .map(e => + path + .relative(publicDataDir, path.join(e.parentPath, e.name)) + .replaceAll(path.sep, '/'), + ) + .sort(); + await writeFile( + path.resolve(publicDir, 'data-file-index.txt'), + files.join('\n') + '\n', + ); +} + +const lootCoreBackend = (): Plugin => ({ + name: 'loot-core-backend', + configureServer(server) { + const child: ChildProcess = spawn( + 'yarn', + [ + 'vite', + 'build', + '--config', + lootCoreConfig, + '--mode', + 'development', + '--watch', + ], + { cwd: lootCoreRoot, stdio: 'inherit' }, + ); + child.on('error', err => { + server.config.logger.error( + `loot-core backend failed to spawn: ${err.message}`, + ); + }); + const cleanup = () => { + if (!child.killed) child.kill('SIGTERM'); + }; + server.httpServer?.once('close', cleanup); + process.once('SIGINT', cleanup); + process.once('SIGTERM', cleanup); + process.once('exit', cleanup); + + server.middlewares.use('/kcab', (req, res, next) => { + const url = new URL(req.url ?? '/', 'http://localhost'); + const filePath = path.join(lootCoreOutDir, url.pathname); + if (!filePath.startsWith(lootCoreOutDir + path.sep)) return next(); + const stream = createReadStream(filePath); + stream + .on('open', () => { + res.setHeader( + 'Content-Type', + CONTENT_TYPES[path.extname(filePath)] ?? 'application/octet-stream', + ); + stream.pipe(res); + }) + .on('error', () => next()); + }); + }, + async closeBundle() { + await mkdir(buildStatsDir, { recursive: true }); + try { + await rename( + path.resolve(__dirname, 'build/kcab/stats.json'), + path.resolve(buildStatsDir, 'loot-core-stats.json'), + ); + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err; + } + }, +}); + +const pluginsServiceAssets = (): Plugin => ({ + name: 'plugins-service-assets', + configureServer(server) { + server.middlewares.use('/service-worker', (req, res, next) => { + const url = new URL(req.url ?? '/', 'http://localhost'); + const filePath = path.join(pluginsServiceDistDir, url.pathname); + if (!filePath.startsWith(pluginsServiceDistDir + path.sep)) return next(); + const stream = createReadStream(filePath); + stream + .on('open', () => { + res.setHeader( + 'Content-Type', + CONTENT_TYPES[path.extname(filePath)] ?? 'application/octet-stream', + ); + stream.pipe(res); + }) + .on('error', () => next()); + }); + }, +}); + +export default defineConfig(async ({ mode, command }) => { const env = loadEnv(mode, process.cwd(), ''); const devHeaders = { 'Cross-Origin-Opener-Policy': 'same-origin', @@ -116,6 +276,32 @@ export default defineConfig(async ({ mode }) => { process.env.REACT_APP_BRANCH = process.env.BRANCH; } + // Electron packaging (--mode=desktop) bundles loot-core directly, so skip + // all browser-only staging there. + if (mode !== 'desktop') { + if (command === 'build') { + const stageKcab = build({ + configFile: lootCoreConfig, + mode: 'production', + root: lootCoreRoot, + }).then(async () => { + const hash = await extractWorkerHash(); + await rm(publicKcabDir, { recursive: true, force: true }); + await cp(lootCoreOutDir, publicKcabDir, { recursive: true }); + return hash; + }); + const [, , hash] = await Promise.all([ + stagePublicData(), + stagePluginsService(), + stageKcab, + ]); + process.env.REACT_APP_BACKEND_WORKER_HASH = hash; + } else { + await stagePublicData(); + process.env.REACT_APP_BACKEND_WORKER_HASH = 'dev'; + } + } + const browserOpen = env.BROWSER_OPEN ? `//${env.BROWSER_OPEN}` : true; return { @@ -213,13 +399,18 @@ export default defineConfig(async ({ mode }) => { }), injectShims(), addWatchers(), + mode === 'desktop' ? undefined : lootCoreBackend(), + mode === 'desktop' ? undefined : pluginsServiceAssets(), react(), babel({ include: [reactCompilerInclude], // n.b. Must be a string to ensure plugin resolution order. See https://github.com/actualbudget/actual/pull/5853 presets: [reactCompilerPreset()], }), - visualizer({ template: 'raw-data' }), + visualizer({ + template: 'raw-data', + filename: 'build-stats/web-stats.json', + }), !!env.HTTPS && basicSsl(), ], test: { diff --git a/packages/loot-core/bin/build-browser b/packages/loot-core/bin/build-browser deleted file mode 100755 index 8758b7fdee..0000000000 --- a/packages/loot-core/bin/build-browser +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -e - -cd `dirname "$0"` -ROOT=`pwd -P` -VITE_ARGS="" - -PUBLIC_DIR="$ROOT"/../../desktop-client/public -DATA_DIR="$PUBLIC_DIR"/data -mkdir -p "$DATA_DIR" -"$ROOT"/copy-migrations "$DATA_DIR" - -cd "$DATA_DIR" -find * -type f | sort > ../data-file-index.txt -cd "$ROOT" - -# Clean out previous build files -rm -f ../lib-dist/browser/* -rm -rf ../../desktop-client/public/kcab - -if [ $NODE_ENV == 'development' ]; then - # In dev mode, always enable watch mode and symlink the build files. - # Make sure to do this before starting the build since watch mode - # will block - VITE_ARGS="$VITE_ARGS --watch" - if [ "$OSTYPE" == "msys" ]; then - # Ensure symlinks are created as native Windows symlinks. - export MSYS=winsymlinks:nativestrict - fi - ln -snf "$ROOT"/../lib-dist/browser ../../desktop-client/public/kcab -fi - -cp ../../../node_modules/@jlongster/sql.js/dist/sql-wasm.wasm "$PUBLIC_DIR"/sql-wasm.wasm - -yarn vite build --config ../vite.config.mts --mode $NODE_ENV $VITE_ARGS - -if [ $NODE_ENV == 'production' ]; then - # In production, just copy the built files - mkdir ../../desktop-client/public/kcab - cp -r ../lib-dist/browser/* ../../desktop-client/public/kcab -fi diff --git a/packages/loot-core/bin/copy-migrations b/packages/loot-core/bin/copy-migrations deleted file mode 100755 index d029d25549..0000000000 --- a/packages/loot-core/bin/copy-migrations +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -e - -ROOT=`dirname $(dirname "$0")` -DEST="$1" - -rm -rf "$DEST"/migrations -mkdir -p "$DEST"/migrations -cp "$ROOT"/migrations/* "$DEST"/migrations/ -cp "$ROOT"/default-db.sqlite "$DEST" diff --git a/packages/loot-core/package.json b/packages/loot-core/package.json index 3d9e50cb04..3499988afe 100644 --- a/packages/loot-core/package.json +++ b/packages/loot-core/package.json @@ -158,8 +158,6 @@ "scripts": { "build:node": "cross-env NODE_ENV=production vite build --config ./vite.desktop.config.mts", "watch:node": "cross-env NODE_ENV=development vite build --config ./vite.desktop.config.mts --watch", - "build:browser": "cross-env NODE_ENV=production ./bin/build-browser", - "watch:browser": "cross-env NODE_ENV=development ./bin/build-browser", "generate:i18n": "i18next", "test": "npm-run-all -cp 'test:*'", "test:node": "ENV=node vitest --run", diff --git a/packages/plugins-service/bin/build-service-worker b/packages/plugins-service/bin/build-service-worker deleted file mode 100755 index 8f378fc98a..0000000000 --- a/packages/plugins-service/bin/build-service-worker +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -e - -cd `dirname "$0"` -ROOT=`pwd -P` -VITE_ARGS="" - -DESKTOP_DIR="$ROOT"/../../desktop-client -SERVICE_WORKER_DIR="$DESKTOP_DIR"/service-worker - -# Clean out previous build files -rm -f ../dist/* -rm -rf "$DESKTOP_DIR"/service-worker - -if [ $NODE_ENV == 'development' ]; then - if [ "$OSTYPE" == "msys" ]; then - # Ensure symlinks are created as native Windows symlinks. - export MSYS=winsymlinks:nativestrict - fi - ln -snf "$ROOT"/../dist/ "$DESKTOP_DIR"/service-worker -fi - -yarn vite build --config ../vite.config.mts --mode $NODE_ENV $VITE_ARGS - -if [ $NODE_ENV == 'production' ]; then - # In production, just copy the built files - mkdir -p "$SERVICE_WORKER_DIR" - cp -r ../dist/* "$DESKTOP_DIR"/service-worker -fi \ No newline at end of file diff --git a/packages/plugins-service/package.json b/packages/plugins-service/package.json index a456806815..d4db1e73d8 100644 --- a/packages/plugins-service/package.json +++ b/packages/plugins-service/package.json @@ -6,9 +6,9 @@ "author": "", "main": "plugin-sw.js", "scripts": { - "build": "cross-env NODE_ENV=production ./bin/build-service-worker", - "build-dev": "cross-env NODE_ENV=development ./bin/build-service-worker", - "watch": "cross-env NODE_ENV=development ./bin/build-service-worker --watch", + "build": "cross-env NODE_ENV=production vite build", + "build-dev": "cross-env NODE_ENV=development vite build", + "watch": "cross-env NODE_ENV=development vite build --watch", "typecheck": "tsgo -b && tsc-strict" }, "dependencies": { diff --git a/upcoming-release-notes/7602.md b/upcoming-release-notes/7602.md new file mode 100644 index 0000000000..dc551a5878 --- /dev/null +++ b/upcoming-release-notes/7602.md @@ -0,0 +1,6 @@ +--- +category: Maintenance +authors: [MatissJanis] +--- + +Build scripts: port browser build version to using lage diff --git a/yarn.lock b/yarn.lock index 7626783485..cee3bed87a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -264,6 +264,7 @@ __metadata: mdast-util-newline-to-break: "npm:^2.0.0" memoize-one: "npm:^6.0.0" pikaday: "npm:1.8.2" + plugins-service: "workspace:*" promise-retry: "npm:^2.0.1" re-resizable: "npm:^6.11.2" react: "npm:19.2.4" @@ -22776,7 +22777,7 @@ __metadata: languageName: node linkType: hard -"plugins-service@workspace:packages/plugins-service": +"plugins-service@workspace:*, plugins-service@workspace:packages/plugins-service": version: 0.0.0-use.local resolution: "plugins-service@workspace:packages/plugins-service" dependencies: