:electron: Embed the sync-server (experimental) (#4526)

* sync server embedded

* cleanup

* remove comment

* remove comment

* changing settings names

* release notes

* release notes

* making dev easier (and slower)

* updating reference to webroot

* using the workspace package yo

* coderabbit

packages/desktop-electron/index.ts

@@ -2,6 +2,7 @@ import fs from 'fs';
 import { createServer, Server } from 'http';
 import path from 'path';
 
+import ngrok from '@ngrok/ngrok';
 import {
   net,
   app,
@@ -19,7 +20,7 @@ import {
   Env,
   ForkOptions,
 } from 'electron';
-import { copy, exists, remove } from 'fs-extra';
+import { copy, exists, mkdir, remove } from 'fs-extra';
 import promiseRetry from 'promise-retry';
 
 import type { GlobalPrefsJson } from '../loot-core/src/types/prefs';
@@ -56,6 +57,7 @@ if (!isDev || !process.env.ACTUAL_DATA_DIR) {
 // be closed automatically when the JavaScript object is garbage collected.
 let clientWin: BrowserWindow | null;
 let serverProcess: UtilityProcess | null;
+let actualServerProcess: UtilityProcess | null;
 
 let oAuthServer: ReturnType<typeof createServer> | null;
 
@@ -63,25 +65,16 @@ let queuedClientWinLogs: string[] = []; // logs that are queued up until the cli
 
 const logMessage = (loglevel: 'info' | 'error', message: string) => {
   // Electron main process logs
-  switch (loglevel) {
-    case 'info':
-      console.info(message);
-      break;
-    case 'error':
-      console.error(message);
-      break;
-  }
+  const trimmedMessage = JSON.stringify(message.trim()); // ensure line endings are removed
+  console[loglevel](trimmedMessage);
 
   if (!clientWin) {
     // queue up the logs until the client window is ready
-    queuedClientWinLogs.push(
-      // eslint-disable-next-line rulesdir/typography
-      `console.${loglevel}('Actual Sync Server Log:', ${JSON.stringify(message)})`,
-    );
+    queuedClientWinLogs.push(`console.${loglevel}(${trimmedMessage})`);
   } else {
     // Send the queued up logs to the devtools console
     clientWin.webContents.executeJavaScript(
-      `console.${loglevel}('Actual Sync Server Log:', ${JSON.stringify(message)})`,
+      `console.${loglevel}(${trimmedMessage})`,
     );
   }
 };
@@ -130,7 +123,7 @@ if (isDev) {
 }
 
 async function loadGlobalPrefs() {
-  let state: GlobalPrefsJson | undefined = undefined;
+  let state: GlobalPrefsJson = {};
   try {
     state = JSON.parse(
       fs.readFileSync(
@@ -152,10 +145,10 @@ async function createBackgroundProcess() {
     ...process.env, // required
   };
 
-  if (globalPrefs?.['server-self-signed-cert']) {
+  if (globalPrefs['server-self-signed-cert']) {
     envVariables = {
       ...envVariables,
-      NODE_EXTRA_CA_CERTS: globalPrefs?.['server-self-signed-cert'], // add self signed cert to env - fetch can pick it up
+      NODE_EXTRA_CA_CERTS: globalPrefs['server-self-signed-cert'], // add self signed cert to env - fetch can pick it up
     };
   }
 
@@ -176,14 +169,12 @@ async function createBackgroundProcess() {
 
   serverProcess.stdout?.on('data', (chunk: Buffer) => {
     // Send the Server log messages to the main browser window
-    clientWin?.webContents.executeJavaScript(`
-      console.info('Server Log:', ${JSON.stringify(chunk.toString('utf8'))})`);
+    logMessage('info', `Server Log: ${chunk.toString('utf8')}`);
   });
 
   serverProcess.stderr?.on('data', (chunk: Buffer) => {
     // Send the Server log messages out to the main browser window
-    clientWin?.webContents.executeJavaScript(`
-      console.error('Server Log:', ${JSON.stringify(chunk.toString('utf8'))})`);
+    logMessage('error', `Server Log: ${chunk.toString('utf8')}`);
   });
 
   serverProcess.on('message', msg => {
@@ -204,6 +195,148 @@ async function createBackgroundProcess() {
   });
 }
 
+async function startSyncServer() {
+  try {
+    const globalPrefs = await loadGlobalPrefs();
+
+    const syncServerConfig = {
+      port: globalPrefs.syncServerConfig?.port || 5007,
+      ACTUAL_SERVER_DATA_DIR: path.resolve(
+        process.env.ACTUAL_DATA_DIR!,
+        'actual-server',
+      ),
+      ACTUAL_SERVER_FILES: path.resolve(
+        process.env.ACTUAL_DATA_DIR!,
+        'actual-server',
+        'server-files',
+      ),
+      ACTUAL_USER_FILES: path.resolve(
+        process.env.ACTUAL_DATA_DIR!,
+        'actual-server',
+        'user-files',
+      ),
+    };
+
+    const serverPath = path.join(
+      // require.resolve will recursively search up the workspace for the module
+      path.dirname(require.resolve('@actual-app/sync-server/package.json')),
+      'app.js',
+    );
+
+    const webRoot = path.join(
+      // require.resolve will recursively search up the workspace for the module
+      path.dirname(require.resolve('@actual-app/web/package.json')),
+      'build',
+    );
+
+    // Use env variables to configure the server
+    const envVariables: Env = {
+      ...process.env, // required
+      ACTUAL_PORT: `${syncServerConfig.port}`,
+      ACTUAL_SERVER_FILES: `${syncServerConfig.ACTUAL_SERVER_FILES}`,
+      ACTUAL_USER_FILES: `${syncServerConfig.ACTUAL_USER_FILES}`,
+      ACTUAL_DATA_DIR: `${syncServerConfig.ACTUAL_SERVER_DATA_DIR}`,
+      ACTUAL_WEB_ROOT: webRoot,
+    };
+
+    // ACTUAL_SERVER_DATA_DIR is the root directory for the sync-server
+    if (!fs.existsSync(syncServerConfig.ACTUAL_SERVER_DATA_DIR)) {
+      mkdir(syncServerConfig.ACTUAL_SERVER_DATA_DIR, { recursive: true });
+    }
+
+    let forkOptions: ForkOptions = {
+      stdio: 'pipe',
+      env: envVariables,
+    };
+
+    if (isDev) {
+      forkOptions = { ...forkOptions, execArgv: ['--inspect'] };
+    }
+
+    let syncServerStarted = false;
+
+    const syncServerPromise = new Promise<void>(resolve => {
+      actualServerProcess = utilityProcess.fork(serverPath, [], forkOptions);
+
+      actualServerProcess.stdout?.on('data', (chunk: Buffer) => {
+        // Send the Server console.log messages to the main browser window
+        logMessage('info', `Sync-Server: ${chunk.toString('utf8')}`);
+      });
+
+      actualServerProcess.stderr?.on('data', (chunk: Buffer) => {
+        // Send the Server console.error messages out to the main browser window
+        logMessage('error', `Sync-Server: ${chunk.toString('utf8')}`);
+      });
+
+      actualServerProcess.on('message', msg => {
+        switch (msg.type) {
+          case 'server-started':
+            logMessage('info', 'Sync-Server: Actual Sync Server has started!');
+            syncServerStarted = true;
+            resolve();
+            break;
+          default:
+            logMessage(
+              'info',
+              'Sync-Server: Unknown server message: ' + msg.type,
+            );
+        }
+      });
+    });
+
+    const SYNC_SERVER_WAIT_TIMEOUT = 20000; // wait 20 seconds for the server to start - if it doesn't, throw an error
+
+    const syncServerTimeout = new Promise<void>((_, reject) => {
+      setTimeout(() => {
+        if (!syncServerStarted) {
+          const errorMessage = `Sync-Server: Failed to start within ${SYNC_SERVER_WAIT_TIMEOUT / 1000} seconds. Something is wrong. Please raise a github issue.`;
+          logMessage('error', errorMessage);
+          reject(new Error(errorMessage));
+        }
+      }, SYNC_SERVER_WAIT_TIMEOUT);
+    });
+
+    return await Promise.race([syncServerPromise, syncServerTimeout]); // Either the server has started or the timeout is reached
+  } catch (error) {
+    logMessage('error', `Sync-Server: Error starting sync server: ${error}`);
+  }
+}
+
+async function exposeSyncServer(
+  syncServerConfig: GlobalPrefsJson['syncServerConfig'],
+) {
+  const hasRequiredConfig =
+    syncServerConfig?.ngrokConfig?.authToken &&
+    syncServerConfig?.ngrokConfig?.domain &&
+    syncServerConfig?.port;
+
+  if (!hasRequiredConfig) {
+    logMessage(
+      'error',
+      'Sync-Server: Cannot expose sync server: missing ngrok settings',
+    );
+    return { error: 'Missing ngrok settings' };
+  }
+
+  try {
+    const listener = await ngrok.forward({
+      schemes: ['https'],
+      addr: syncServerConfig.port,
+      authtoken: syncServerConfig?.ngrokConfig?.authToken,
+      domain: syncServerConfig?.ngrokConfig?.domain,
+    });
+
+    logMessage(
+      'info',
+      `Sync-Server: Exposing actual server on url: ${listener.url()}`,
+    );
+    return { url: listener.url() };
+  } catch (error) {
+    logMessage('error', `Unable to run ngrok: ${error}`);
+    return { error: `Unable to run ngrok. ${error}` };
+  }
+}
+
 async function createWindow() {
   const windowState = await getWindowState();
 
@@ -342,6 +475,17 @@ app.on('ready', async () => {
   // Install an `app://` protocol that always returns the base HTML
   // file no matter what URL it is. This allows us to use react-router
   // on the frontend
+
+  const globalPrefs = await loadGlobalPrefs();
+
+  if (globalPrefs.syncServerConfig?.autoStart) {
+    // wait for both server and ngrok to start before starting the Actual client to ensure server is available
+    await Promise.allSettled([
+      startSyncServer(),
+      exposeSyncServer(globalPrefs.syncServerConfig),
+    ]);
+  }
+
   protocol.handle('app', request => {
     if (request.method !== 'GET') {
       return new Response(null, {

packages/desktop-electron/package.json

@@ -48,7 +48,7 @@
         "flatpak",
         "AppImage"
       ],
-      "artifactName": "${productName}-linux.${ext}"
+      "artifactName": "${productName}-linux-${arch}.${ext}"
     },
     "flatpak": {
       "runtimeVersion": "23.08",
@@ -85,6 +85,8 @@
     "npmRebuild": false
   },
   "dependencies": {
+    "@actual-app/sync-server": "workspace:*",
+    "@ngrok/ngrok": "^1.4.1",
     "better-sqlite3": "^11.7.0",
     "fs-extra": "^11.3.0",
     "promise-retry": "^2.0.1"

packages/loot-core/src/types/prefs.d.ts

@@ -89,6 +89,15 @@ export type GlobalPrefs = Partial<{
   preferredDarkTheme: DarkTheme;
   documentDir: string; // Electron only
   serverSelfSignedCert: string; // Electron only
+  syncServerConfig?: {
+    // Electron only
+    autoStart?: boolean;
+    port?: number;
+    ngrokConfig?: {
+      domain?: string;
+      authToken?: string;
+    };
+  };
 }>;
 
 // GlobalPrefsJson represents what's saved in the global-store.json file
@@ -109,6 +118,7 @@ export type GlobalPrefsJson = Partial<{
   theme?: GlobalPrefs['theme'];
   'preferred-dark-theme'?: GlobalPrefs['preferredDarkTheme'];
   'server-self-signed-cert'?: GlobalPrefs['serverSelfSignedCert'];
+  syncServerConfig?: GlobalPrefs['syncServerConfig'];
 }>;
 
 export type AuthMethods = 'password' | 'openid';

packages/sync-server/src/app.js

@@ -136,6 +136,9 @@ export async function run() {
     app.listen(config.get('port'), config.get('hostname'));
   }
 
+  // Signify to any parent process that the server has started. Used in electron desktop app
+  process.parentPort?.postMessage({ type: 'server-started' });
+
   console.log(
     'Listening on ' + config.get('hostname') + ':' + config.get('port') + '...',
   );