diff --git a/bin/package-electron b/bin/package-electron index dde72dc4d6..2ab7be3805 100755 --- a/bin/package-electron +++ b/bin/package-electron @@ -34,21 +34,23 @@ if [ "$OSTYPE" == "msys" ]; then fi fi -yarn workspace loot-core build:node - # Get translations echo "Updating translations..." if ! [ -d packages/desktop-client/locale ]; then git clone https://github.com/actualbudget/translations packages/desktop-client/locale fi pushd packages/desktop-client/locale > /dev/null -git checkout . git pull popd > /dev/null packages/desktop-client/bin/remove-untranslated-languages +yarn workspace loot-core build:node yarn workspace @actual-app/web build --mode=desktop # electron specific build +# required for running the sync-server server +yarn workspace loot-core build:browser +yarn workspace @actual-app/web build:browser + yarn workspace desktop-electron update-client ( diff --git a/package.json b/package.json index 54ee96a290..1e6fdf496c 100644 --- a/package.json +++ b/package.json @@ -22,11 +22,12 @@ "start:server": "yarn workspace @actual-app/sync-server start", "start:server-monitor": "yarn workspace @actual-app/sync-server start-monitor", "start:server-dev": "NODE_ENV=development BROWSER_OPEN=localhost:5006 yarn npm-run-all --parallel 'start:server-monitor' 'start'", - "start:desktop": "yarn rebuild-electron && npm-run-all --parallel 'start:desktop-*'", + "start:desktop": "yarn desktop-dependencies && npm-run-all --parallel 'start:desktop-*'", + "desktop-dependencies": "yarn rebuild-electron && yarn workspace loot-core build:browser", "start:desktop-node": "yarn workspace loot-core watch:node", "start:desktop-client": "yarn workspace @actual-app/web watch", + "start:desktop-server-client": "yarn workspace @actual-app/web build:browser", "start:desktop-electron": "yarn workspace desktop-electron watch", - "start:electron": "yarn start:desktop", "start:browser": "npm-run-all --parallel 'start:browser-*'", "start:browser-backend": "yarn workspace loot-core watch:browser", "start:browser-frontend": "yarn workspace @actual-app/web start:browser", diff --git a/packages/desktop-electron/index.ts b/packages/desktop-electron/index.ts index efd037f731..aedab61023 100644 --- a/packages/desktop-electron/index.ts +++ b/packages/desktop-electron/index.ts @@ -2,6 +2,7 @@ import fs from 'fs'; import { createServer, Server } from 'http'; import path from 'path'; +import ngrok from '@ngrok/ngrok'; import { net, app, @@ -19,7 +20,7 @@ import { Env, ForkOptions, } from 'electron'; -import { copy, exists, remove } from 'fs-extra'; +import { copy, exists, mkdir, remove } from 'fs-extra'; import promiseRetry from 'promise-retry'; import type { GlobalPrefsJson } from '../loot-core/src/types/prefs'; @@ -56,6 +57,7 @@ if (!isDev || !process.env.ACTUAL_DATA_DIR) { // be closed automatically when the JavaScript object is garbage collected. let clientWin: BrowserWindow | null; let serverProcess: UtilityProcess | null; +let actualServerProcess: UtilityProcess | null; let oAuthServer: ReturnType | null; @@ -63,25 +65,16 @@ let queuedClientWinLogs: string[] = []; // logs that are queued up until the cli const logMessage = (loglevel: 'info' | 'error', message: string) => { // Electron main process logs - switch (loglevel) { - case 'info': - console.info(message); - break; - case 'error': - console.error(message); - break; - } + const trimmedMessage = JSON.stringify(message.trim()); // ensure line endings are removed + console[loglevel](trimmedMessage); if (!clientWin) { // queue up the logs until the client window is ready - queuedClientWinLogs.push( - // eslint-disable-next-line rulesdir/typography - `console.${loglevel}('Actual Sync Server Log:', ${JSON.stringify(message)})`, - ); + queuedClientWinLogs.push(`console.${loglevel}(${trimmedMessage})`); } else { // Send the queued up logs to the devtools console clientWin.webContents.executeJavaScript( - `console.${loglevel}('Actual Sync Server Log:', ${JSON.stringify(message)})`, + `console.${loglevel}(${trimmedMessage})`, ); } }; @@ -130,7 +123,7 @@ if (isDev) { } async function loadGlobalPrefs() { - let state: GlobalPrefsJson | undefined = undefined; + let state: GlobalPrefsJson = {}; try { state = JSON.parse( fs.readFileSync( @@ -152,10 +145,10 @@ async function createBackgroundProcess() { ...process.env, // required }; - if (globalPrefs?.['server-self-signed-cert']) { + if (globalPrefs['server-self-signed-cert']) { envVariables = { ...envVariables, - NODE_EXTRA_CA_CERTS: globalPrefs?.['server-self-signed-cert'], // add self signed cert to env - fetch can pick it up + NODE_EXTRA_CA_CERTS: globalPrefs['server-self-signed-cert'], // add self signed cert to env - fetch can pick it up }; } @@ -176,14 +169,12 @@ async function createBackgroundProcess() { serverProcess.stdout?.on('data', (chunk: Buffer) => { // Send the Server log messages to the main browser window - clientWin?.webContents.executeJavaScript(` - console.info('Server Log:', ${JSON.stringify(chunk.toString('utf8'))})`); + logMessage('info', `Server Log: ${chunk.toString('utf8')}`); }); serverProcess.stderr?.on('data', (chunk: Buffer) => { // Send the Server log messages out to the main browser window - clientWin?.webContents.executeJavaScript(` - console.error('Server Log:', ${JSON.stringify(chunk.toString('utf8'))})`); + logMessage('error', `Server Log: ${chunk.toString('utf8')}`); }); serverProcess.on('message', msg => { @@ -204,6 +195,148 @@ async function createBackgroundProcess() { }); } +async function startSyncServer() { + try { + const globalPrefs = await loadGlobalPrefs(); + + const syncServerConfig = { + port: globalPrefs.syncServerConfig?.port || 5007, + ACTUAL_SERVER_DATA_DIR: path.resolve( + process.env.ACTUAL_DATA_DIR!, + 'actual-server', + ), + ACTUAL_SERVER_FILES: path.resolve( + process.env.ACTUAL_DATA_DIR!, + 'actual-server', + 'server-files', + ), + ACTUAL_USER_FILES: path.resolve( + process.env.ACTUAL_DATA_DIR!, + 'actual-server', + 'user-files', + ), + }; + + const serverPath = path.join( + // require.resolve will recursively search up the workspace for the module + path.dirname(require.resolve('@actual-app/sync-server/package.json')), + 'app.js', + ); + + const webRoot = path.join( + // require.resolve will recursively search up the workspace for the module + path.dirname(require.resolve('@actual-app/web/package.json')), + 'build', + ); + + // Use env variables to configure the server + const envVariables: Env = { + ...process.env, // required + ACTUAL_PORT: `${syncServerConfig.port}`, + ACTUAL_SERVER_FILES: `${syncServerConfig.ACTUAL_SERVER_FILES}`, + ACTUAL_USER_FILES: `${syncServerConfig.ACTUAL_USER_FILES}`, + ACTUAL_DATA_DIR: `${syncServerConfig.ACTUAL_SERVER_DATA_DIR}`, + ACTUAL_WEB_ROOT: webRoot, + }; + + // ACTUAL_SERVER_DATA_DIR is the root directory for the sync-server + if (!fs.existsSync(syncServerConfig.ACTUAL_SERVER_DATA_DIR)) { + mkdir(syncServerConfig.ACTUAL_SERVER_DATA_DIR, { recursive: true }); + } + + let forkOptions: ForkOptions = { + stdio: 'pipe', + env: envVariables, + }; + + if (isDev) { + forkOptions = { ...forkOptions, execArgv: ['--inspect'] }; + } + + let syncServerStarted = false; + + const syncServerPromise = new Promise(resolve => { + actualServerProcess = utilityProcess.fork(serverPath, [], forkOptions); + + actualServerProcess.stdout?.on('data', (chunk: Buffer) => { + // Send the Server console.log messages to the main browser window + logMessage('info', `Sync-Server: ${chunk.toString('utf8')}`); + }); + + actualServerProcess.stderr?.on('data', (chunk: Buffer) => { + // Send the Server console.error messages out to the main browser window + logMessage('error', `Sync-Server: ${chunk.toString('utf8')}`); + }); + + actualServerProcess.on('message', msg => { + switch (msg.type) { + case 'server-started': + logMessage('info', 'Sync-Server: Actual Sync Server has started!'); + syncServerStarted = true; + resolve(); + break; + default: + logMessage( + 'info', + 'Sync-Server: Unknown server message: ' + msg.type, + ); + } + }); + }); + + const SYNC_SERVER_WAIT_TIMEOUT = 20000; // wait 20 seconds for the server to start - if it doesn't, throw an error + + const syncServerTimeout = new Promise((_, reject) => { + setTimeout(() => { + if (!syncServerStarted) { + const errorMessage = `Sync-Server: Failed to start within ${SYNC_SERVER_WAIT_TIMEOUT / 1000} seconds. Something is wrong. Please raise a github issue.`; + logMessage('error', errorMessage); + reject(new Error(errorMessage)); + } + }, SYNC_SERVER_WAIT_TIMEOUT); + }); + + return await Promise.race([syncServerPromise, syncServerTimeout]); // Either the server has started or the timeout is reached + } catch (error) { + logMessage('error', `Sync-Server: Error starting sync server: ${error}`); + } +} + +async function exposeSyncServer( + syncServerConfig: GlobalPrefsJson['syncServerConfig'], +) { + const hasRequiredConfig = + syncServerConfig?.ngrokConfig?.authToken && + syncServerConfig?.ngrokConfig?.domain && + syncServerConfig?.port; + + if (!hasRequiredConfig) { + logMessage( + 'error', + 'Sync-Server: Cannot expose sync server: missing ngrok settings', + ); + return { error: 'Missing ngrok settings' }; + } + + try { + const listener = await ngrok.forward({ + schemes: ['https'], + addr: syncServerConfig.port, + authtoken: syncServerConfig?.ngrokConfig?.authToken, + domain: syncServerConfig?.ngrokConfig?.domain, + }); + + logMessage( + 'info', + `Sync-Server: Exposing actual server on url: ${listener.url()}`, + ); + return { url: listener.url() }; + } catch (error) { + logMessage('error', `Unable to run ngrok: ${error}`); + return { error: `Unable to run ngrok. ${error}` }; + } +} + async function createWindow() { const windowState = await getWindowState(); @@ -342,6 +475,17 @@ app.on('ready', async () => { // Install an `app://` protocol that always returns the base HTML // file no matter what URL it is. This allows us to use react-router // on the frontend + + const globalPrefs = await loadGlobalPrefs(); + + if (globalPrefs.syncServerConfig?.autoStart) { + // wait for both server and ngrok to start before starting the Actual client to ensure server is available + await Promise.allSettled([ + startSyncServer(), + exposeSyncServer(globalPrefs.syncServerConfig), + ]); + } + protocol.handle('app', request => { if (request.method !== 'GET') { return new Response(null, { diff --git a/packages/desktop-electron/package.json b/packages/desktop-electron/package.json index fbefae9720..edebed32b4 100644 --- a/packages/desktop-electron/package.json +++ b/packages/desktop-electron/package.json @@ -48,7 +48,7 @@ "flatpak", "AppImage" ], - "artifactName": "${productName}-linux.${ext}" + "artifactName": "${productName}-linux-${arch}.${ext}" }, "flatpak": { "runtimeVersion": "23.08", @@ -85,6 +85,8 @@ "npmRebuild": false }, "dependencies": { + "@actual-app/sync-server": "workspace:*", + "@ngrok/ngrok": "^1.4.1", "better-sqlite3": "^11.7.0", "fs-extra": "^11.3.0", "promise-retry": "^2.0.1" diff --git a/packages/loot-core/src/types/prefs.d.ts b/packages/loot-core/src/types/prefs.d.ts index 8b6dacc0f1..68f848eb53 100644 --- a/packages/loot-core/src/types/prefs.d.ts +++ b/packages/loot-core/src/types/prefs.d.ts @@ -89,6 +89,15 @@ export type GlobalPrefs = Partial<{ preferredDarkTheme: DarkTheme; documentDir: string; // Electron only serverSelfSignedCert: string; // Electron only + syncServerConfig?: { + // Electron only + autoStart?: boolean; + port?: number; + ngrokConfig?: { + domain?: string; + authToken?: string; + }; + }; }>; // GlobalPrefsJson represents what's saved in the global-store.json file @@ -109,6 +118,7 @@ export type GlobalPrefsJson = Partial<{ theme?: GlobalPrefs['theme']; 'preferred-dark-theme'?: GlobalPrefs['preferredDarkTheme']; 'server-self-signed-cert'?: GlobalPrefs['serverSelfSignedCert']; + syncServerConfig?: GlobalPrefs['syncServerConfig']; }>; export type AuthMethods = 'password' | 'openid'; diff --git a/packages/sync-server/src/app.js b/packages/sync-server/src/app.js index 0dda48dd90..bb91e6ab16 100644 --- a/packages/sync-server/src/app.js +++ b/packages/sync-server/src/app.js @@ -136,6 +136,9 @@ export async function run() { app.listen(config.get('port'), config.get('hostname')); } + // Signify to any parent process that the server has started. Used in electron desktop app + process.parentPort?.postMessage({ type: 'server-started' }); + console.log( 'Listening on ' + config.get('hostname') + ':' + config.get('port') + '...', ); diff --git a/upcoming-release-notes/4526.md b/upcoming-release-notes/4526.md new file mode 100644 index 0000000000..32e06cf0bc --- /dev/null +++ b/upcoming-release-notes/4526.md @@ -0,0 +1,6 @@ +--- +category: Features +authors: [MikesGlitch] +--- + +Experimental: Embedding the sync server into the desktop app diff --git a/yarn.lock b/yarn.lock index eabc4e5aca..9543e00f18 100644 --- a/yarn.lock +++ b/yarn.lock @@ -83,7 +83,7 @@ __metadata: languageName: unknown linkType: soft -"@actual-app/sync-server@workspace:packages/sync-server": +"@actual-app/sync-server@workspace:*, @actual-app/sync-server@workspace:packages/sync-server": version: 0.0.0-use.local resolution: "@actual-app/sync-server@workspace:packages/sync-server" dependencies: @@ -2878,6 +2878,145 @@ __metadata: languageName: node linkType: hard +"@ngrok/ngrok-android-arm-eabi@npm:1.4.1": + version: 1.4.1 + resolution: "@ngrok/ngrok-android-arm-eabi@npm:1.4.1" + conditions: os=android & cpu=arm + languageName: node + linkType: hard + +"@ngrok/ngrok-android-arm64@npm:1.4.1": + version: 1.4.1 + resolution: "@ngrok/ngrok-android-arm64@npm:1.4.1" + conditions: os=android & cpu=arm64 + languageName: node + linkType: hard + +"@ngrok/ngrok-darwin-arm64@npm:1.4.1": + version: 1.4.1 + resolution: "@ngrok/ngrok-darwin-arm64@npm:1.4.1" + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + +"@ngrok/ngrok-darwin-universal@npm:1.4.1": + version: 1.4.1 + resolution: "@ngrok/ngrok-darwin-universal@npm:1.4.1" + conditions: os=darwin + languageName: node + linkType: hard + +"@ngrok/ngrok-darwin-x64@npm:1.4.1": + version: 1.4.1 + resolution: "@ngrok/ngrok-darwin-x64@npm:1.4.1" + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + +"@ngrok/ngrok-freebsd-x64@npm:1.4.1": + version: 1.4.1 + resolution: "@ngrok/ngrok-freebsd-x64@npm:1.4.1" + conditions: os=freebsd & cpu=x64 + languageName: node + linkType: hard + +"@ngrok/ngrok-linux-arm-gnueabihf@npm:1.4.1": + version: 1.4.1 + resolution: "@ngrok/ngrok-linux-arm-gnueabihf@npm:1.4.1" + conditions: os=linux & cpu=arm + languageName: node + linkType: hard + +"@ngrok/ngrok-linux-arm64-gnu@npm:1.4.1": + version: 1.4.1 + resolution: "@ngrok/ngrok-linux-arm64-gnu@npm:1.4.1" + conditions: os=linux & cpu=arm64 & libc=glibc + languageName: node + linkType: hard + +"@ngrok/ngrok-linux-arm64-musl@npm:1.4.1": + version: 1.4.1 + resolution: "@ngrok/ngrok-linux-arm64-musl@npm:1.4.1" + conditions: os=linux & cpu=arm64 & libc=musl + languageName: node + linkType: hard + +"@ngrok/ngrok-linux-x64-gnu@npm:1.4.1": + version: 1.4.1 + resolution: "@ngrok/ngrok-linux-x64-gnu@npm:1.4.1" + conditions: os=linux & cpu=x64 & libc=glibc + languageName: node + linkType: hard + +"@ngrok/ngrok-linux-x64-musl@npm:1.4.1": + version: 1.4.1 + resolution: "@ngrok/ngrok-linux-x64-musl@npm:1.4.1" + conditions: os=linux & cpu=x64 & libc=musl + languageName: node + linkType: hard + +"@ngrok/ngrok-win32-ia32-msvc@npm:1.4.1": + version: 1.4.1 + resolution: "@ngrok/ngrok-win32-ia32-msvc@npm:1.4.1" + conditions: os=win32 & cpu=ia32 + languageName: node + linkType: hard + +"@ngrok/ngrok-win32-x64-msvc@npm:1.4.1": + version: 1.4.1 + resolution: "@ngrok/ngrok-win32-x64-msvc@npm:1.4.1" + conditions: os=win32 & cpu=x64 + languageName: node + linkType: hard + +"@ngrok/ngrok@npm:^1.4.1": + version: 1.4.1 + resolution: "@ngrok/ngrok@npm:1.4.1" + dependencies: + "@ngrok/ngrok-android-arm-eabi": "npm:1.4.1" + "@ngrok/ngrok-android-arm64": "npm:1.4.1" + "@ngrok/ngrok-darwin-arm64": "npm:1.4.1" + "@ngrok/ngrok-darwin-universal": "npm:1.4.1" + "@ngrok/ngrok-darwin-x64": "npm:1.4.1" + "@ngrok/ngrok-freebsd-x64": "npm:1.4.1" + "@ngrok/ngrok-linux-arm-gnueabihf": "npm:1.4.1" + "@ngrok/ngrok-linux-arm64-gnu": "npm:1.4.1" + "@ngrok/ngrok-linux-arm64-musl": "npm:1.4.1" + "@ngrok/ngrok-linux-x64-gnu": "npm:1.4.1" + "@ngrok/ngrok-linux-x64-musl": "npm:1.4.1" + "@ngrok/ngrok-win32-ia32-msvc": "npm:1.4.1" + "@ngrok/ngrok-win32-x64-msvc": "npm:1.4.1" + dependenciesMeta: + "@ngrok/ngrok-android-arm-eabi": + optional: true + "@ngrok/ngrok-android-arm64": + optional: true + "@ngrok/ngrok-darwin-arm64": + optional: true + "@ngrok/ngrok-darwin-universal": + optional: true + "@ngrok/ngrok-darwin-x64": + optional: true + "@ngrok/ngrok-freebsd-x64": + optional: true + "@ngrok/ngrok-linux-arm-gnueabihf": + optional: true + "@ngrok/ngrok-linux-arm64-gnu": + optional: true + "@ngrok/ngrok-linux-arm64-musl": + optional: true + "@ngrok/ngrok-linux-x64-gnu": + optional: true + "@ngrok/ngrok-linux-x64-musl": + optional: true + "@ngrok/ngrok-win32-ia32-msvc": + optional: true + "@ngrok/ngrok-win32-x64-msvc": + optional: true + checksum: 10/c86956756af6eb9f2cc47aba19ac14f59a0d031defc52ee6845b17363b051213abd29c984f11501d88a482f343fbadb0e7bf855803068bceb96bf5fb16dfb2cb + languageName: node + linkType: hard + "@nodelib/fs.scandir@npm:2.1.5": version: 2.1.5 resolution: "@nodelib/fs.scandir@npm:2.1.5" @@ -9403,8 +9542,10 @@ __metadata: version: 0.0.0-use.local resolution: "desktop-electron@workspace:packages/desktop-electron" dependencies: + "@actual-app/sync-server": "workspace:*" "@electron/notarize": "npm:2.4.0" "@electron/rebuild": "npm:3.6.0" + "@ngrok/ngrok": "npm:^1.4.1" "@types/copyfiles": "npm:^2" "@types/fs-extra": "npm:^11" better-sqlite3: "npm:^11.7.0"