:electron: Embed the sync-server (experimental) (#4526)

* sync server embedded

* cleanup

* remove comment

* remove comment

* changing settings names

* release notes

* release notes

* making dev easier (and slower)

* updating reference to webroot

* using the workspace package yo

* coderabbit

bin/package-electron

@@ -34,21 +34,23 @@ if [ "$OSTYPE" == "msys" ]; then
     fi
 fi
 
-yarn workspace loot-core build:node
-
 # Get translations
 echo "Updating translations..."
 if ! [ -d packages/desktop-client/locale ]; then
     git clone https://github.com/actualbudget/translations packages/desktop-client/locale
 fi
 pushd packages/desktop-client/locale > /dev/null
-git checkout .
 git pull
 popd > /dev/null
 packages/desktop-client/bin/remove-untranslated-languages
 
+yarn workspace loot-core build:node
 yarn workspace @actual-app/web build --mode=desktop # electron specific build
 
+# required for running the sync-server server
+yarn workspace loot-core build:browser
+yarn workspace @actual-app/web build:browser
+
 yarn workspace desktop-electron update-client
 
 (

package.json

@@ -22,11 +22,12 @@
     "start:server": "yarn workspace @actual-app/sync-server start",
     "start:server-monitor": "yarn workspace @actual-app/sync-server start-monitor",
     "start:server-dev": "NODE_ENV=development BROWSER_OPEN=localhost:5006 yarn npm-run-all --parallel 'start:server-monitor' 'start'",
-    "start:desktop": "yarn rebuild-electron && npm-run-all --parallel 'start:desktop-*'",
+    "start:desktop": "yarn desktop-dependencies && npm-run-all --parallel 'start:desktop-*'",
+    "desktop-dependencies": "yarn rebuild-electron && yarn workspace loot-core build:browser",
     "start:desktop-node": "yarn workspace loot-core watch:node",
     "start:desktop-client": "yarn workspace @actual-app/web watch",
+    "start:desktop-server-client": "yarn workspace @actual-app/web build:browser",
     "start:desktop-electron": "yarn workspace desktop-electron watch",
-    "start:electron": "yarn start:desktop",
     "start:browser": "npm-run-all --parallel 'start:browser-*'",
     "start:browser-backend": "yarn workspace loot-core watch:browser",
     "start:browser-frontend": "yarn workspace @actual-app/web start:browser",

packages/desktop-electron/index.ts

@@ -2,6 +2,7 @@ import fs from 'fs';
 import { createServer, Server } from 'http';
 import path from 'path';
 
+import ngrok from '@ngrok/ngrok';
 import {
   net,
   app,
@@ -19,7 +20,7 @@ import {
   Env,
   ForkOptions,
 } from 'electron';
-import { copy, exists, remove } from 'fs-extra';
+import { copy, exists, mkdir, remove } from 'fs-extra';
 import promiseRetry from 'promise-retry';
 
 import type { GlobalPrefsJson } from '../loot-core/src/types/prefs';
@@ -56,6 +57,7 @@ if (!isDev || !process.env.ACTUAL_DATA_DIR) {
 // be closed automatically when the JavaScript object is garbage collected.
 let clientWin: BrowserWindow | null;
 let serverProcess: UtilityProcess | null;
+let actualServerProcess: UtilityProcess | null;
 
 let oAuthServer: ReturnType<typeof createServer> | null;
 
@@ -63,25 +65,16 @@ let queuedClientWinLogs: string[] = []; // logs that are queued up until the cli
 
 const logMessage = (loglevel: 'info' | 'error', message: string) => {
   // Electron main process logs
-  switch (loglevel) {
-    case 'info':
-      console.info(message);
-      break;
-    case 'error':
-      console.error(message);
-      break;
-  }
+  const trimmedMessage = JSON.stringify(message.trim()); // ensure line endings are removed
+  console[loglevel](trimmedMessage);
 
   if (!clientWin) {
     // queue up the logs until the client window is ready
-    queuedClientWinLogs.push(
-      // eslint-disable-next-line rulesdir/typography
-      `console.${loglevel}('Actual Sync Server Log:', ${JSON.stringify(message)})`,
-    );
+    queuedClientWinLogs.push(`console.${loglevel}(${trimmedMessage})`);
   } else {
     // Send the queued up logs to the devtools console
     clientWin.webContents.executeJavaScript(
-      `console.${loglevel}('Actual Sync Server Log:', ${JSON.stringify(message)})`,
+      `console.${loglevel}(${trimmedMessage})`,
     );
   }
 };
@@ -130,7 +123,7 @@ if (isDev) {
 }
 
 async function loadGlobalPrefs() {
-  let state: GlobalPrefsJson | undefined = undefined;
+  let state: GlobalPrefsJson = {};
   try {
     state = JSON.parse(
       fs.readFileSync(
@@ -152,10 +145,10 @@ async function createBackgroundProcess() {
     ...process.env, // required
   };
 
-  if (globalPrefs?.['server-self-signed-cert']) {
+  if (globalPrefs['server-self-signed-cert']) {
     envVariables = {
       ...envVariables,
-      NODE_EXTRA_CA_CERTS: globalPrefs?.['server-self-signed-cert'], // add self signed cert to env - fetch can pick it up
+      NODE_EXTRA_CA_CERTS: globalPrefs['server-self-signed-cert'], // add self signed cert to env - fetch can pick it up
     };
   }
 
@@ -176,14 +169,12 @@ async function createBackgroundProcess() {
 
   serverProcess.stdout?.on('data', (chunk: Buffer) => {
     // Send the Server log messages to the main browser window
-    clientWin?.webContents.executeJavaScript(`
-      console.info('Server Log:', ${JSON.stringify(chunk.toString('utf8'))})`);
+    logMessage('info', `Server Log: ${chunk.toString('utf8')}`);
   });
 
   serverProcess.stderr?.on('data', (chunk: Buffer) => {
     // Send the Server log messages out to the main browser window
-    clientWin?.webContents.executeJavaScript(`
-      console.error('Server Log:', ${JSON.stringify(chunk.toString('utf8'))})`);
+    logMessage('error', `Server Log: ${chunk.toString('utf8')}`);
   });
 
   serverProcess.on('message', msg => {
@@ -204,6 +195,148 @@ async function createBackgroundProcess() {
   });
 }
 
+async function startSyncServer() {
+  try {
+    const globalPrefs = await loadGlobalPrefs();
+
+    const syncServerConfig = {
+      port: globalPrefs.syncServerConfig?.port || 5007,
+      ACTUAL_SERVER_DATA_DIR: path.resolve(
+        process.env.ACTUAL_DATA_DIR!,
+        'actual-server',
+      ),
+      ACTUAL_SERVER_FILES: path.resolve(
+        process.env.ACTUAL_DATA_DIR!,
+        'actual-server',
+        'server-files',
+      ),
+      ACTUAL_USER_FILES: path.resolve(
+        process.env.ACTUAL_DATA_DIR!,
+        'actual-server',
+        'user-files',
+      ),
+    };
+
+    const serverPath = path.join(
+      // require.resolve will recursively search up the workspace for the module
+      path.dirname(require.resolve('@actual-app/sync-server/package.json')),
+      'app.js',
+    );
+
+    const webRoot = path.join(
+      // require.resolve will recursively search up the workspace for the module
+      path.dirname(require.resolve('@actual-app/web/package.json')),
+      'build',
+    );
+
+    // Use env variables to configure the server
+    const envVariables: Env = {
+      ...process.env, // required
+      ACTUAL_PORT: `${syncServerConfig.port}`,
+      ACTUAL_SERVER_FILES: `${syncServerConfig.ACTUAL_SERVER_FILES}`,
+      ACTUAL_USER_FILES: `${syncServerConfig.ACTUAL_USER_FILES}`,
+      ACTUAL_DATA_DIR: `${syncServerConfig.ACTUAL_SERVER_DATA_DIR}`,
+      ACTUAL_WEB_ROOT: webRoot,
+    };
+
+    // ACTUAL_SERVER_DATA_DIR is the root directory for the sync-server
+    if (!fs.existsSync(syncServerConfig.ACTUAL_SERVER_DATA_DIR)) {
+      mkdir(syncServerConfig.ACTUAL_SERVER_DATA_DIR, { recursive: true });
+    }
+
+    let forkOptions: ForkOptions = {
+      stdio: 'pipe',
+      env: envVariables,
+    };
+
+    if (isDev) {
+      forkOptions = { ...forkOptions, execArgv: ['--inspect'] };
+    }
+
+    let syncServerStarted = false;
+
+    const syncServerPromise = new Promise<void>(resolve => {
+      actualServerProcess = utilityProcess.fork(serverPath, [], forkOptions);
+
+      actualServerProcess.stdout?.on('data', (chunk: Buffer) => {
+        // Send the Server console.log messages to the main browser window
+        logMessage('info', `Sync-Server: ${chunk.toString('utf8')}`);
+      });
+
+      actualServerProcess.stderr?.on('data', (chunk: Buffer) => {
+        // Send the Server console.error messages out to the main browser window
+        logMessage('error', `Sync-Server: ${chunk.toString('utf8')}`);
+      });
+
+      actualServerProcess.on('message', msg => {
+        switch (msg.type) {
+          case 'server-started':
+            logMessage('info', 'Sync-Server: Actual Sync Server has started!');
+            syncServerStarted = true;
+            resolve();
+            break;
+          default:
+            logMessage(
+              'info',
+              'Sync-Server: Unknown server message: ' + msg.type,
+            );
+        }
+      });
+    });
+
+    const SYNC_SERVER_WAIT_TIMEOUT = 20000; // wait 20 seconds for the server to start - if it doesn't, throw an error
+
+    const syncServerTimeout = new Promise<void>((_, reject) => {
+      setTimeout(() => {
+        if (!syncServerStarted) {
+          const errorMessage = `Sync-Server: Failed to start within ${SYNC_SERVER_WAIT_TIMEOUT / 1000} seconds. Something is wrong. Please raise a github issue.`;
+          logMessage('error', errorMessage);
+          reject(new Error(errorMessage));
+        }
+      }, SYNC_SERVER_WAIT_TIMEOUT);
+    });
+
+    return await Promise.race([syncServerPromise, syncServerTimeout]); // Either the server has started or the timeout is reached
+  } catch (error) {
+    logMessage('error', `Sync-Server: Error starting sync server: ${error}`);
+  }
+}
+
+async function exposeSyncServer(
+  syncServerConfig: GlobalPrefsJson['syncServerConfig'],
+) {
+  const hasRequiredConfig =
+    syncServerConfig?.ngrokConfig?.authToken &&
+    syncServerConfig?.ngrokConfig?.domain &&
+    syncServerConfig?.port;
+
+  if (!hasRequiredConfig) {
+    logMessage(
+      'error',
+      'Sync-Server: Cannot expose sync server: missing ngrok settings',
+    );
+    return { error: 'Missing ngrok settings' };
+  }
+
+  try {
+    const listener = await ngrok.forward({
+      schemes: ['https'],
+      addr: syncServerConfig.port,
+      authtoken: syncServerConfig?.ngrokConfig?.authToken,
+      domain: syncServerConfig?.ngrokConfig?.domain,
+    });
+
+    logMessage(
+      'info',
+      `Sync-Server: Exposing actual server on url: ${listener.url()}`,
+    );
+    return { url: listener.url() };
+  } catch (error) {
+    logMessage('error', `Unable to run ngrok: ${error}`);
+    return { error: `Unable to run ngrok. ${error}` };
+  }
+}
+
 async function createWindow() {
   const windowState = await getWindowState();
 
@@ -342,6 +475,17 @@ app.on('ready', async () => {
   // Install an `app://` protocol that always returns the base HTML
   // file no matter what URL it is. This allows us to use react-router
   // on the frontend
+
+  const globalPrefs = await loadGlobalPrefs();
+
+  if (globalPrefs.syncServerConfig?.autoStart) {
+    // wait for both server and ngrok to start before starting the Actual client to ensure server is available
+    await Promise.allSettled([
+      startSyncServer(),
+      exposeSyncServer(globalPrefs.syncServerConfig),
+    ]);
+  }
+
   protocol.handle('app', request => {
     if (request.method !== 'GET') {
       return new Response(null, {

packages/desktop-electron/package.json

@@ -48,7 +48,7 @@
         "flatpak",
         "AppImage"
       ],
-      "artifactName": "${productName}-linux.${ext}"
+      "artifactName": "${productName}-linux-${arch}.${ext}"
     },
     "flatpak": {
       "runtimeVersion": "23.08",
@@ -85,6 +85,8 @@
     "npmRebuild": false
   },
   "dependencies": {
+    "@actual-app/sync-server": "workspace:*",
+    "@ngrok/ngrok": "^1.4.1",
     "better-sqlite3": "^11.7.0",
     "fs-extra": "^11.3.0",
     "promise-retry": "^2.0.1"

packages/loot-core/src/types/prefs.d.ts

@@ -89,6 +89,15 @@ export type GlobalPrefs = Partial<{
   preferredDarkTheme: DarkTheme;
   documentDir: string; // Electron only
   serverSelfSignedCert: string; // Electron only
+  syncServerConfig?: {
+    // Electron only
+    autoStart?: boolean;
+    port?: number;
+    ngrokConfig?: {
+      domain?: string;
+      authToken?: string;
+    };
+  };
 }>;
 
 // GlobalPrefsJson represents what's saved in the global-store.json file
@@ -109,6 +118,7 @@ export type GlobalPrefsJson = Partial<{
   theme?: GlobalPrefs['theme'];
   'preferred-dark-theme'?: GlobalPrefs['preferredDarkTheme'];
   'server-self-signed-cert'?: GlobalPrefs['serverSelfSignedCert'];
+  syncServerConfig?: GlobalPrefs['syncServerConfig'];
 }>;
 
 export type AuthMethods = 'password' | 'openid';

packages/sync-server/src/app.js

@@ -136,6 +136,9 @@ export async function run() {
     app.listen(config.get('port'), config.get('hostname'));
   }
 
+  // Signify to any parent process that the server has started. Used in electron desktop app
+  process.parentPort?.postMessage({ type: 'server-started' });
+
   console.log(
     'Listening on ' + config.get('hostname') + ':' + config.get('port') + '...',
   );

upcoming-release-notes/4526.md

@@ -0,0 +1,6 @@
+---
+category: Features
+authors: [MikesGlitch]
+---
+
+Experimental: Embedding the sync server into the desktop app

yarn.lock

@@ -83,7 +83,7 @@ __metadata:
   languageName: unknown
   linkType: soft
 
-"@actual-app/sync-server@workspace:packages/sync-server":
+"@actual-app/sync-server@workspace:*, @actual-app/sync-server@workspace:packages/sync-server":
   version: 0.0.0-use.local
   resolution: "@actual-app/sync-server@workspace:packages/sync-server"
   dependencies:
@@ -2878,6 +2878,145 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@ngrok/ngrok-android-arm-eabi@npm:1.4.1":
+  version: 1.4.1
+  resolution: "@ngrok/ngrok-android-arm-eabi@npm:1.4.1"
+  conditions: os=android & cpu=arm
+  languageName: node
+  linkType: hard
+
+"@ngrok/ngrok-android-arm64@npm:1.4.1":
+  version: 1.4.1
+  resolution: "@ngrok/ngrok-android-arm64@npm:1.4.1"
+  conditions: os=android & cpu=arm64
+  languageName: node
+  linkType: hard
+
+"@ngrok/ngrok-darwin-arm64@npm:1.4.1":
+  version: 1.4.1
+  resolution: "@ngrok/ngrok-darwin-arm64@npm:1.4.1"
+  conditions: os=darwin & cpu=arm64
+  languageName: node
+  linkType: hard
+
+"@ngrok/ngrok-darwin-universal@npm:1.4.1":
+  version: 1.4.1
+  resolution: "@ngrok/ngrok-darwin-universal@npm:1.4.1"
+  conditions: os=darwin
+  languageName: node
+  linkType: hard
+
+"@ngrok/ngrok-darwin-x64@npm:1.4.1":
+  version: 1.4.1
+  resolution: "@ngrok/ngrok-darwin-x64@npm:1.4.1"
+  conditions: os=darwin & cpu=x64
+  languageName: node
+  linkType: hard
+
+"@ngrok/ngrok-freebsd-x64@npm:1.4.1":
+  version: 1.4.1
+  resolution: "@ngrok/ngrok-freebsd-x64@npm:1.4.1"
+  conditions: os=freebsd & cpu=x64
+  languageName: node
+  linkType: hard
+
+"@ngrok/ngrok-linux-arm-gnueabihf@npm:1.4.1":
+  version: 1.4.1
+  resolution: "@ngrok/ngrok-linux-arm-gnueabihf@npm:1.4.1"
+  conditions: os=linux & cpu=arm
+  languageName: node
+  linkType: hard
+
+"@ngrok/ngrok-linux-arm64-gnu@npm:1.4.1":
+  version: 1.4.1
+  resolution: "@ngrok/ngrok-linux-arm64-gnu@npm:1.4.1"
+  conditions: os=linux & cpu=arm64 & libc=glibc
+  languageName: node
+  linkType: hard
+
+"@ngrok/ngrok-linux-arm64-musl@npm:1.4.1":
+  version: 1.4.1
+  resolution: "@ngrok/ngrok-linux-arm64-musl@npm:1.4.1"
+  conditions: os=linux & cpu=arm64 & libc=musl
+  languageName: node
+  linkType: hard
+
+"@ngrok/ngrok-linux-x64-gnu@npm:1.4.1":
+  version: 1.4.1
+  resolution: "@ngrok/ngrok-linux-x64-gnu@npm:1.4.1"
+  conditions: os=linux & cpu=x64 & libc=glibc
+  languageName: node
+  linkType: hard
+
+"@ngrok/ngrok-linux-x64-musl@npm:1.4.1":
+  version: 1.4.1
+  resolution: "@ngrok/ngrok-linux-x64-musl@npm:1.4.1"
+  conditions: os=linux & cpu=x64 & libc=musl
+  languageName: node
+  linkType: hard
+
+"@ngrok/ngrok-win32-ia32-msvc@npm:1.4.1":
+  version: 1.4.1
+  resolution: "@ngrok/ngrok-win32-ia32-msvc@npm:1.4.1"
+  conditions: os=win32 & cpu=ia32
+  languageName: node
+  linkType: hard
+
+"@ngrok/ngrok-win32-x64-msvc@npm:1.4.1":
+  version: 1.4.1
+  resolution: "@ngrok/ngrok-win32-x64-msvc@npm:1.4.1"
+  conditions: os=win32 & cpu=x64
+  languageName: node
+  linkType: hard
+
+"@ngrok/ngrok@npm:^1.4.1":
+  version: 1.4.1
+  resolution: "@ngrok/ngrok@npm:1.4.1"
+  dependencies:
+    "@ngrok/ngrok-android-arm-eabi": "npm:1.4.1"
+    "@ngrok/ngrok-android-arm64": "npm:1.4.1"
+    "@ngrok/ngrok-darwin-arm64": "npm:1.4.1"
+    "@ngrok/ngrok-darwin-universal": "npm:1.4.1"
+    "@ngrok/ngrok-darwin-x64": "npm:1.4.1"
+    "@ngrok/ngrok-freebsd-x64": "npm:1.4.1"
+    "@ngrok/ngrok-linux-arm-gnueabihf": "npm:1.4.1"
+    "@ngrok/ngrok-linux-arm64-gnu": "npm:1.4.1"
+    "@ngrok/ngrok-linux-arm64-musl": "npm:1.4.1"
+    "@ngrok/ngrok-linux-x64-gnu": "npm:1.4.1"
+    "@ngrok/ngrok-linux-x64-musl": "npm:1.4.1"
+    "@ngrok/ngrok-win32-ia32-msvc": "npm:1.4.1"
+    "@ngrok/ngrok-win32-x64-msvc": "npm:1.4.1"
+  dependenciesMeta:
+    "@ngrok/ngrok-android-arm-eabi":
+      optional: true
+    "@ngrok/ngrok-android-arm64":
+      optional: true
+    "@ngrok/ngrok-darwin-arm64":
+      optional: true
+    "@ngrok/ngrok-darwin-universal":
+      optional: true
+    "@ngrok/ngrok-darwin-x64":
+      optional: true
+    "@ngrok/ngrok-freebsd-x64":
+      optional: true
+    "@ngrok/ngrok-linux-arm-gnueabihf":
+      optional: true
+    "@ngrok/ngrok-linux-arm64-gnu":
+      optional: true
+    "@ngrok/ngrok-linux-arm64-musl":
+      optional: true
+    "@ngrok/ngrok-linux-x64-gnu":
+      optional: true
+    "@ngrok/ngrok-linux-x64-musl":
+      optional: true
+    "@ngrok/ngrok-win32-ia32-msvc":
+      optional: true
+    "@ngrok/ngrok-win32-x64-msvc":
+      optional: true
+  checksum: 10/c86956756af6eb9f2cc47aba19ac14f59a0d031defc52ee6845b17363b051213abd29c984f11501d88a482f343fbadb0e7bf855803068bceb96bf5fb16dfb2cb
+  languageName: node
+  linkType: hard
+
 "@nodelib/fs.scandir@npm:2.1.5":
   version: 2.1.5
   resolution: "@nodelib/fs.scandir@npm:2.1.5"
@@ -9403,8 +9542,10 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "desktop-electron@workspace:packages/desktop-electron"
   dependencies:
+    "@actual-app/sync-server": "workspace:*"
     "@electron/notarize": "npm:2.4.0"
     "@electron/rebuild": "npm:3.6.0"
+    "@ngrok/ngrok": "npm:^1.4.1"
     "@types/copyfiles": "npm:^2"
     "@types/fs-extra": "npm:^11"
     better-sqlite3: "npm:^11.7.0"