:electron: Playwright testing for electron (#4674)

* playwright testing for electron

* pipeline updates

* fix normal e2e scripts

* fix path to artifact

* listing out whats there

* attempt to fix ci

* umm

* again

* setting a viewport

* window size to be consistent across machine for tests

* now it work... Righhttt?

* hmm

* do it

* worflow

* will this work

* oops

* dont skip

* trust in the pipeline gods

* remove update snapshots, just do it in the pipeline

* change name of snapshot to account for os

* lint

* fix package script

.github/workflows/e2e-test.yml

@@ -48,6 +48,27 @@ jobs:
           path: packages/desktop-client/test-results/
           retention-days: 30
           overwrite: true
+
+  functional-desktop-app:
+    name: Functional Desktop App
+    runs-on: ubuntu-latest
+    container:
+      image: mcr.microsoft.com/playwright:v1.41.1-jammy
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up environment
+        uses: ./.github/actions/setup
+      - name: Run Desktop app E2E Tests
+        run: |
+          xvfb-run --auto-servernum --server-args="-screen 0 1920x1080x24" -- yarn e2e:desktop
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: desktop-app-test-results
+          path: packages/desktop-electron/e2e/test-results/
+          retention-days: 30
+          overwrite: true
+
   vrt:
     name: Visual regression
     needs: netlify

.github/workflows/update-vrt.yml

@@ -1,7 +1,7 @@
 name: /update-vrt
 on:
   issue_comment:
-    types: [ created ]
+    types: [created]
 
 permissions:
   pull-requests: read
@@ -22,7 +22,7 @@ jobs:
       image: mcr.microsoft.com/playwright:v1.41.1-jammy
     steps:
       - name: Get PR branch
-#        Until https://github.com/xt0rted/pull-request-comment-branch/issues/322 is resolved we use the forked version
+        #        Until https://github.com/xt0rted/pull-request-comment-branch/issues/322 is resolved we use the forked version
         uses: gotson/pull-request-comment-branch@head-repo-owner-dist
         id: comment-branch
       - uses: actions/checkout@v4
@@ -97,7 +97,7 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           commentId: ${{ github.event.comment.id }}
-          reaction: "rocket"
+          reaction: 'rocket'
 
   add-starting-reaction:
     runs-on: ubuntu-latest
@@ -112,4 +112,4 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           commentId: ${{ github.event.comment.id }}
-          reaction: "+1"
+          reaction: '+1'

bin/package-electron

@@ -6,8 +6,8 @@ RELEASE=""
 CI=${CI:-false}
 
 cd "$ROOT/.."
-
 POSITIONAL=()
+SKIP_EXE_BUILD=false
 while [[ $# -gt 0 ]]; do
   key="$1"
 
@@ -16,23 +16,18 @@ while [[ $# -gt 0 ]]; do
       RELEASE="production"
       shift
       ;;
+      --skip-exe-build)
+      SKIP_EXE_BUILD=true
+      shift
+      ;;
       *)
       POSITIONAL+=("$1")
       shift
       ;;
   esac
 done
-set -- "${POSITIONAL[@]}"
 
-if [ "$OSTYPE" == "msys" ]; then
-    if [ $CI != true ]; then
-        read -s -p "Windows certificate password: " -r CSC_KEY_PASSWORD
-        export CSC_KEY_PASSWORD
-    elif [ -n "$CIRCLE_TAG" ]; then
-        # We only want to run this on CircleCI as Github doesn't have the CSC_KEY_PASSWORD secret set.
-        certutil -f -p ${CSC_KEY_PASSWORD} -importPfx ~/windows-shift-reset-llc.p12
-    fi
-fi
+set -- "${POSITIONAL[@]}"
 
 # Get translations
 echo "Updating translations..."
@@ -57,14 +52,20 @@ yarn workspace desktop-electron update-client
     cd packages/desktop-electron;
     yarn clean;
 
-    if [ "$RELEASE" == "production" ]; then
-        if [ -f ../../.secret-tokens ]; then
-            source ../../.secret-tokens
-        fi
-        yarn build
-
-        echo "\nCreated release"
+    if [ $SKIP_EXE_BUILD == true ]; then
+        echo "Building the dist"
+        yarn build:dist
+        echo "Skipping exe build"
     else
-        SKIP_NOTARIZATION=true yarn build
+      if [ "$RELEASE" == "production" ]; then
+          if [ -f ../../.secret-tokens ]; then
+              source ../../.secret-tokens
+          fi
+          yarn build
+
+          echo "Created release"
+      else
+          SKIP_NOTARIZATION=true yarn build
+      fi
     fi
 )

package.json

@@ -39,7 +39,8 @@
     "generate:release-notes": "ts-node ./bin/release-note-generator.ts",
     "test": "yarn workspaces foreach --all --parallel --verbose run test",
     "test:debug": "yarn workspaces foreach --all --verbose run test",
-    "e2e": "yarn workspaces foreach --all --parallel --verbose run e2e",
+    "e2e": "yarn workspaces foreach --all --exclude desktop-electron --parallel --verbose run e2e",
+    "e2e:desktop": "yarn build:desktop --skip-exe-build && yarn workspace desktop-electron e2e",
     "vrt": "yarn workspaces foreach --all --parallel --verbose run vrt",
     "vrt:docker": "./bin/run-vrt",
     "rebuild-electron": "./node_modules/.bin/electron-rebuild -f -m ./packages/loot-core",

packages/desktop-electron/.gitignore

@@ -0,0 +1,5 @@
+# testing
+test-results
+playwright-report
+data/
+!data/.gitkeep

packages/desktop-electron/e2e/onboarding.test.ts

@@ -0,0 +1,20 @@
+import { test, expect, _electron } from '@playwright/test';
+
+test.describe('Onboarding', () => {
+  test('checks the page visuals', async () => {
+    const electronApp = await _electron.launch({
+      args: ['.'],
+      env: {
+        ...process.env,
+        ACTUAL_DOCUMENT_DIR: 'e2e/data',
+        ACTUAL_DATA_DIR: 'e2e/data',
+        EXECUTION_CONTEXT: 'playwright',
+        NODE_ENV: 'development',
+      },
+    });
+
+    const window = await electronApp.firstWindow();
+    await expect(window).toHaveScreenshot();
+    await electronApp.close();
+  });
+});

packages/desktop-electron/index.ts

@@ -33,7 +33,8 @@ import {
 
 import './security';
 
-const isDev = !app.isPackaged; // dev mode if not packaged
+const isPlaywrightTest = process.env.EXECUTION_CONTEXT === 'playwright';
+const isDev = !isPlaywrightTest && !app.isPackaged; // dev mode if not packaged and not playwright
 
 process.env.lootCoreScript = isDev
   ? 'loot-core/lib-dist/electron/bundle.desktop.js' // serve from local output in development (provides hot-reloading)
@@ -45,12 +46,20 @@ protocol.registerSchemesAsPrivileged([
   { scheme: 'app', privileges: { standard: true } },
 ]);
 
-if (!isDev || !process.env.ACTUAL_DOCUMENT_DIR) {
-  process.env.ACTUAL_DOCUMENT_DIR = app.getPath('documents');
-}
+if (isPlaywrightTest) {
+  if (!process.env.ACTUAL_DOCUMENT_DIR || !process.env.ACTUAL_DATA_DIR) {
+    throw new Error(
+      'ACTUAL_DOCUMENT_DIR and ACTUAL_DATA_DIR must be set in the environment for playwright tests',
+    );
+  }
+} else {
+  if (!isDev || !process.env.ACTUAL_DOCUMENT_DIR) {
+    process.env.ACTUAL_DOCUMENT_DIR = app.getPath('documents');
+  }
 
-if (!isDev || !process.env.ACTUAL_DATA_DIR) {
-  process.env.ACTUAL_DATA_DIR = app.getPath('userData');
+  if (!isDev || !process.env.ACTUAL_DATA_DIR) {
+    process.env.ACTUAL_DATA_DIR = app.getPath('userData');
+  }
 }
 
 // Keep a global reference of the window object, if you don't, the window will

packages/desktop-electron/package.json

@@ -10,7 +10,8 @@
     "build": "yarn build:dist && electron-builder",
     "build:dist": "tsc --p tsconfig.dist.json && yarn copy-static-assets",
     "copy-static-assets": "copyfiles --exclude 'build/**/*' **/*.html icons/**/* build",
-    "watch": "yarn build:dist && cross-env ACTUAL_DOCUMENT_DIR=\"../../data\" ACTUAL_DATA_DIR=\"../../data\" electron ."
+    "watch": "yarn build:dist && cross-env ACTUAL_DOCUMENT_DIR=\"../../data\" ACTUAL_DATA_DIR=\"../../data\" electron .",
+    "e2e": "npx playwright test"
   },
   "main": "build/index.js",
   "build": {
@@ -97,6 +98,7 @@
   "devDependencies": {
     "@electron/notarize": "2.4.0",
     "@electron/rebuild": "3.6.0",
+    "@playwright/test": "1.41.2",
     "@types/copyfiles": "^2",
     "@types/fs-extra": "^11",
     "copyfiles": "^2.4.1",

packages/desktop-electron/playwright.config.ts

@@ -0,0 +1,22 @@
+import { defineConfig } from '@playwright/test';
+
+// eslint-disable-next-line import/no-default-export
+export default defineConfig({
+  timeout: 45000, // 45 seconds
+  retries: 1,
+  testDir: 'e2e/',
+  reporter: undefined,
+  outputDir: 'e2e/test-results/',
+  snapshotPathTemplate:
+    '{testDir}/__screenshots__/{testFilePath}/{arg}-{platform}{ext}',
+  use: {
+    userAgent: 'playwright',
+    screenshot: 'on',
+    browserName: 'chromium',
+    trace: 'on-first-retry',
+    ignoreHTTPSErrors: true,
+  },
+  expect: {
+    toHaveScreenshot: { maxDiffPixels: 5 },
+  },
+});

packages/desktop-electron/tsconfig.dist.json

@@ -11,5 +11,5 @@
     "outDir": "build"
   },
   "include": ["."],
-  "exclude": ["**/node_modules/*", "build/**/*"]
+  "exclude": ["**/node_modules/*", "build/**/*", "e2e/**/*"]
 }

packages/desktop-electron/window-state.ts

@@ -130,6 +130,16 @@ function validateState(state?: WindowState): Partial<WindowState> {
 }
 
 export async function get() {
+  if (process.env.EXECUTION_CONTEXT === 'playwright') {
+    // For Playwright screenshots to be consistent across machine we need a fixed window size
+    return {
+      x: 100,
+      y: 50,
+      width: 1300,
+      height: 800,
+    };
+  }
+
   const screen = electron.screen;
   const displayBounds = screen.getPrimaryDisplay().bounds;
 

upcoming-release-notes/4674.md

@@ -0,0 +1,6 @@
+---
+category: Maintenance
+authors: [MikesGlitch]
+---
+
+Added initial Playwright tests for Electron

yarn.lock

@@ -9718,6 +9718,7 @@ __metadata:
     "@electron/notarize": "npm:2.4.0"
     "@electron/rebuild": "npm:3.6.0"
     "@ngrok/ngrok": "npm:^1.4.1"
+    "@playwright/test": "npm:1.41.2"
     "@types/copyfiles": "npm:^2"
     "@types/fs-extra": "npm:^11"
     better-sqlite3: "npm:^11.9.1"