diff --git a/.github/actions/netlify-wait-for-build b/.github/actions/netlify-wait-for-build index c3cb9540cc..8ab96e3f16 100755 --- a/.github/actions/netlify-wait-for-build +++ b/.github/actions/netlify-wait-for-build @@ -9,8 +9,8 @@ function get_status() { curl --header "Authorization: Bearer $GITHUB_TOKEN" "https://api.github.com/repos/actualbudget/actual/commits/$COMMIT_SHA/statuses" > /tmp/status.json cat /tmp/status.json echo "::endgroup::" - netlify=$(jq '[.[] | select(.context == "netlify/actualbudget/deploy-preview")][0]' /tmp/status.json) - state=$(jq -r '.state' <<< "$netlify") + netlify=$(yarn jq '[.[] | select(.context == "netlify/actualbudget/deploy-preview")][0]' /tmp/status.json) + state=$(yarn jq -r '.state' <<< "$netlify") echo "::group::Netlify Status" echo "$netlify" echo "::endgroup::" @@ -32,7 +32,7 @@ done if [ "$state" == "success" ]; then echo -e "\033[0;32mNetlify build succeeded!\033[0m" - jq -r '"url=" + .target_url' <<< "$netlify" > $GITHUB_OUTPUT + yarn jq -r '"url=" + .target_url' <<< "$netlify" > $GITHUB_OUTPUT exit 0 else echo -e "\033[0;31mNetlify build failed. Cancelling end-to-end tests.\033[0m" diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 58bba08297..e452d08506 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -10,15 +10,15 @@ concurrency: cancel-in-progress: true jobs: - test: - name: Run end-to-end tests on Netlify PR preview + functional: + name: Functional runs-on: ubuntu-latest + container: + image: mcr.microsoft.com/playwright:v1.37.0-jammy steps: - uses: actions/checkout@v3 - name: Set up environment uses: ./.github/actions/setup - - name: Setup Playwright - run: npx playwright install chromium --with-deps - name: Wait for Netlify build to finish id: netlify env: @@ -35,3 +35,28 @@ jobs: name: desktop-client-test-results path: packages/desktop-client/test-results/ retention-days: 30 + vrt: + name: Visual regression + runs-on: ubuntu-latest + container: + image: mcr.microsoft.com/playwright:v1.37.0-jammy + steps: + - uses: actions/checkout@v3 + - name: Set up environment + uses: ./.github/actions/setup + - name: Wait for Netlify build to finish + id: netlify + env: + COMMIT_SHA: ${{ github.event.pull_request.head.sha }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: ./.github/actions/netlify-wait-for-build + - name: Run VRT Tests on Netlify URL + run: yarn vrt + env: + E2E_START_URL: ${{ steps.netlify.outputs.url }} + - uses: actions/upload-artifact@v3 + if: always() + with: + name: desktop-client-test-results + path: packages/desktop-client/test-results/ + retention-days: 30 diff --git a/package.json b/package.json index 3b0eee9ca3..5b495a1fae 100644 --- a/package.json +++ b/package.json @@ -33,11 +33,13 @@ "test": "yarn workspaces foreach --parallel --verbose run test", "test:debug": "yarn workspaces foreach --verbose run test", "e2e": "yarn workspaces foreach --parallel --verbose run e2e", + "vrt": "yarn workspaces foreach --parallel --verbose run vrt", "rebuild-electron": "./node_modules/.bin/electron-rebuild -f -m ./packages/loot-core", "rebuild-node": "yarn workspace loot-core rebuild", "lint": "eslint . --max-warnings 0", "lint:verbose": "DEBUG=eslint:cli-engine eslint . --max-warnings 0", "typecheck": "yarn tsc", + "jq": "./node_modules/node-jq/bin/jq", "postinstall": "patch-package" }, "devDependencies": { @@ -49,6 +51,7 @@ "eslint-plugin-prettier": "4.2.1", "eslint-plugin-react": "7.32.2", "eslint-plugin-rulesdir": "^0.2.2", + "node-jq": "^4.0.1", "npm-run-all": "^4.1.3", "patch-package": "^6.1.2", "prettier": "2.8.2", diff --git a/packages/desktop-client/README.md b/packages/desktop-client/README.md index d061bfb07f..3228051f1d 100644 --- a/packages/desktop-client/README.md +++ b/packages/desktop-client/README.md @@ -4,11 +4,13 @@ Actual on the web E2E (end-to-end) tests use [Playwright](https://playwright.dev/). Running them requires an Actual server to be running either locally or on a remote server. +### Functional + Running against the local server: ```sh # Start the development server -yarn start:browser +yarn start # Run against the local server (localhost:3001) yarn e2e @@ -19,3 +21,37 @@ Running against a remote server: ```sh E2E_START_URL=http://my-remote-server.com yarn e2e ``` + +### Visual regression + +Visual regression tests (also known as screenshot tests) check that the visual appearance of the product has not regressed. Each environment has slightly different colors, fonts etc. Mac differs from Windows which differs from Linux. In order to have a stable test environment for visual comparisons - you must use a standartised docker container. This ensures that the tests are always ran in a consistent environment. + +Prerequisites: + +- Docker installed + +#### Running against the local server + +First start the dev server: + +```sh +HTTPS=true yarn start +``` + +Next, run the standartised docker container and launch the visual regression tests from within it. + +```sh +# Run docker container +docker run --rm --network host -v $(pwd):/work/ -w /work/ -it mcr.microsoft.com/playwright:v1.37.0-jammy /bin/bash + +# Run the VRT tests: important - they MUST be ran against a HTTPS server +E2E_START_URL=https://192.168.0.178:3001 yarn vrt +``` + +#### Running against a remote server + +You can also run the tests against a remote server by passing the URL: + +```sh +E2E_START_URL=https://my-remote-server.com yarn vrt +``` diff --git a/packages/desktop-client/e2e/accounts.test.js b/packages/desktop-client/e2e/accounts.test.js index 631cc30701..878e3c3542 100644 --- a/packages/desktop-client/e2e/accounts.test.js +++ b/packages/desktop-client/e2e/accounts.test.js @@ -2,6 +2,7 @@ import { test, expect } from '@playwright/test'; import { ConfigurationPage } from './page-models/configuration-page'; import { Navigation } from './page-models/navigation'; +import screenshotConfig from './screenshot.config'; test.describe('Accounts', () => { let page; @@ -28,13 +29,13 @@ test.describe('Accounts', () => { balance: 100, }); - expect(await accountPage.getNthTransaction(0)).toMatchObject({ - payee: 'Starting Balance', - notes: '', - category: 'Starting Balances', - debit: '', - credit: '100.00', - }); + const transaction = accountPage.getNthTransaction(0); + await expect(transaction.payee).toHaveText('Starting Balance'); + await expect(transaction.notes).toHaveText(''); + await expect(transaction.category).toHaveText('Starting Balances'); + await expect(transaction.debit).toHaveText(''); + await expect(transaction.credit).toHaveText('100.00'); + await expect(page).toHaveScreenshot(screenshotConfig(page)); }); test('closes an account', async () => { @@ -44,8 +45,10 @@ test.describe('Accounts', () => { const modal = await accountPage.clickCloseAccount(); await modal.selectTransferAccount('Vanguard 401k'); + await expect(page).toHaveScreenshot(screenshotConfig(page)); await modal.closeAccount(); await expect(accountPage.accountName).toHaveText('Closed: Roth IRA'); + await expect(page).toHaveScreenshot(screenshotConfig(page)); }); }); diff --git a/packages/desktop-client/e2e/accounts.test.js-snapshots/Accounts-closes-an-account-1-chromium-linux.png b/packages/desktop-client/e2e/accounts.test.js-snapshots/Accounts-closes-an-account-1-chromium-linux.png new file mode 100644 index 0000000000..2134f58218 Binary files /dev/null and b/packages/desktop-client/e2e/accounts.test.js-snapshots/Accounts-closes-an-account-1-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/accounts.test.js-snapshots/Accounts-closes-an-account-2-chromium-linux.png b/packages/desktop-client/e2e/accounts.test.js-snapshots/Accounts-closes-an-account-2-chromium-linux.png new file mode 100644 index 0000000000..412753803f Binary files /dev/null and b/packages/desktop-client/e2e/accounts.test.js-snapshots/Accounts-closes-an-account-2-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/accounts.test.js-snapshots/Accounts-creates-a-new-account-and-views-the-initial-balance-transaction-1-chromium-linux.png b/packages/desktop-client/e2e/accounts.test.js-snapshots/Accounts-creates-a-new-account-and-views-the-initial-balance-transaction-1-chromium-linux.png new file mode 100644 index 0000000000..f91b967e00 Binary files /dev/null and b/packages/desktop-client/e2e/accounts.test.js-snapshots/Accounts-creates-a-new-account-and-views-the-initial-balance-transaction-1-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/budget.test.js b/packages/desktop-client/e2e/budget.test.js index ebfd78d005..f868c582a3 100644 --- a/packages/desktop-client/e2e/budget.test.js +++ b/packages/desktop-client/e2e/budget.test.js @@ -1,6 +1,7 @@ import { test, expect } from '@playwright/test'; import { ConfigurationPage } from './page-models/configuration-page'; +import screenshotConfig from './screenshot.config'; test.describe('Budget', () => { let page; @@ -13,6 +14,11 @@ test.describe('Budget', () => { await page.goto('/'); budgetPage = await configurationPage.createTestFile(); + + // Move mouse to corner of the screen; + // sometimes the mouse hovers on a budget element thus rendering an input box + // and this breaks screenshot tests + await page.mouse.move(0, 0); }); test.afterAll(async () => { @@ -28,6 +34,7 @@ test.describe('Budget', () => { await expect(summary.getByText(/^Overspent in /)).toBeVisible(); await expect(summary.getByText('Budgeted')).toBeVisible(); await expect(summary.getByText('For Next Month')).toBeVisible(); + await expect(page).toHaveScreenshot(screenshotConfig(page)); }); test('transfer funds to another category', async () => { @@ -40,6 +47,7 @@ test.describe('Budget', () => { expect(await budgetPage.getBalanceForRow(2)).toEqual( currentFundsA + currentFundsB, ); + await expect(page).toHaveScreenshot(screenshotConfig(page)); }); test('budget table is rendered', async () => { diff --git a/packages/desktop-client/e2e/budget.test.js-snapshots/Budget-renders-the-summary-information-available-funds-overspent-budgeted-and-for-next-month-1-chromium-linux.png b/packages/desktop-client/e2e/budget.test.js-snapshots/Budget-renders-the-summary-information-available-funds-overspent-budgeted-and-for-next-month-1-chromium-linux.png new file mode 100644 index 0000000000..c66cbca273 Binary files /dev/null and b/packages/desktop-client/e2e/budget.test.js-snapshots/Budget-renders-the-summary-information-available-funds-overspent-budgeted-and-for-next-month-1-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/budget.test.js-snapshots/Budget-transfer-funds-to-another-category-1-chromium-linux.png b/packages/desktop-client/e2e/budget.test.js-snapshots/Budget-transfer-funds-to-another-category-1-chromium-linux.png new file mode 100644 index 0000000000..218831959c Binary files /dev/null and b/packages/desktop-client/e2e/budget.test.js-snapshots/Budget-transfer-funds-to-another-category-1-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/mobile.test.js b/packages/desktop-client/e2e/mobile.test.js index d39d0cfb1e..4f68619a60 100644 --- a/packages/desktop-client/e2e/mobile.test.js +++ b/packages/desktop-client/e2e/mobile.test.js @@ -2,6 +2,7 @@ import { test, expect } from '@playwright/test'; import { ConfigurationPage } from './page-models/configuration-page'; import { MobileNavigation } from './page-models/mobile-navigation'; +import screenshotConfig from './screenshot.config'; test.describe('Mobile', () => { let page; @@ -43,6 +44,7 @@ test.describe('Mobile', () => { 'Water', 'Power', ]); + await expect(page).toHaveScreenshot(screenshotConfig(page)); }); test('opens the accounts page and asserts on balances', async () => { @@ -50,8 +52,9 @@ test.describe('Mobile', () => { const account = await accountsPage.getNthAccount(0); - expect(account.name).toEqual('Ally Savings'); - expect(account.balance).toBeGreaterThan(0); + await expect(account.name).toHaveText('Ally Savings'); + await expect(account.balance).toHaveText('7,653.00'); + await expect(page).toHaveScreenshot(screenshotConfig(page)); }); test('opens individual account page and checks that filtering is working', async () => { @@ -62,17 +65,21 @@ test.describe('Mobile', () => { expect(await accountPage.getBalance()).toBeGreaterThan(0); await expect(accountPage.noTransactionsFoundError).not.toBeVisible(); + await expect(page).toHaveScreenshot(screenshotConfig(page)); await accountPage.searchByText('nothing should be found'); await expect(accountPage.noTransactionsFoundError).toBeVisible(); await expect(accountPage.transactions).toHaveCount(0); + await expect(page).toHaveScreenshot(screenshotConfig(page)); await accountPage.searchByText('Kroger'); await expect(accountPage.transactions).not.toHaveCount(0); + await expect(page).toHaveScreenshot(screenshotConfig(page)); }); test('creates a transaction via footer button', async () => { const transactionEntryPage = await navigation.goToTransactionEntryPage(); + await expect(page).toHaveScreenshot(screenshotConfig(page)); await expect(transactionEntryPage.header).toHaveText('New Transaction'); @@ -89,12 +96,14 @@ test.describe('Mobile', () => { page.getByTestId('account-field'), 'Ally Savings', ); + await expect(page).toHaveScreenshot(screenshotConfig(page)); const accountPage = await transactionEntryPage.createTransaction(); await expect(accountPage.transactions.nth(0)).toHaveText( 'KrogerClothing-12.34', ); + await expect(page).toHaveScreenshot(screenshotConfig(page)); }); test('creates a transaction from `/accounts/:id` page', async () => { @@ -103,6 +112,7 @@ test.describe('Mobile', () => { const transactionEntryPage = await accountPage.clickCreateTransaction(); await expect(transactionEntryPage.header).toHaveText('New Transaction'); + await expect(page).toHaveScreenshot(screenshotConfig(page)); await transactionEntryPage.amountField.fill('12.34'); await transactionEntryPage.fillField( @@ -123,6 +133,7 @@ test.describe('Mobile', () => { test('checks that settings page can be opened', async () => { const settingsPage = await navigation.goToSettingsPage(); + await expect(page).toHaveScreenshot(screenshotConfig(page)); const downloadPromise = page.waitForEvent('download'); @@ -133,5 +144,6 @@ test.describe('Mobile', () => { expect(await download.suggestedFilename()).toMatch( /^\d{4}-\d{2}-\d{2}-.*.zip$/, ); + await expect(page).toHaveScreenshot(screenshotConfig(page)); }); }); diff --git a/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-checks-that-settings-page-can-be-opened-1-chromium-linux.png b/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-checks-that-settings-page-can-be-opened-1-chromium-linux.png new file mode 100644 index 0000000000..dd2abcee93 Binary files /dev/null and b/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-checks-that-settings-page-can-be-opened-1-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-checks-that-settings-page-can-be-opened-2-chromium-linux.png b/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-checks-that-settings-page-can-be-opened-2-chromium-linux.png new file mode 100644 index 0000000000..9e420b859e Binary files /dev/null and b/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-checks-that-settings-page-can-be-opened-2-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-creates-a-transaction-from-accounts-id-page-1-chromium-linux.png b/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-creates-a-transaction-from-accounts-id-page-1-chromium-linux.png new file mode 100644 index 0000000000..722e9b77a3 Binary files /dev/null and b/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-creates-a-transaction-from-accounts-id-page-1-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-creates-a-transaction-via-footer-button-1-chromium-linux.png b/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-creates-a-transaction-via-footer-button-1-chromium-linux.png new file mode 100644 index 0000000000..aa0d6f8c86 Binary files /dev/null and b/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-creates-a-transaction-via-footer-button-1-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-creates-a-transaction-via-footer-button-2-chromium-linux.png b/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-creates-a-transaction-via-footer-button-2-chromium-linux.png new file mode 100644 index 0000000000..d8a7a694ec Binary files /dev/null and b/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-creates-a-transaction-via-footer-button-2-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-creates-a-transaction-via-footer-button-3-chromium-linux.png b/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-creates-a-transaction-via-footer-button-3-chromium-linux.png new file mode 100644 index 0000000000..cb2f733857 Binary files /dev/null and b/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-creates-a-transaction-via-footer-button-3-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-loads-the-budget-page-with-budgeted-amounts-1-chromium-linux.png b/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-loads-the-budget-page-with-budgeted-amounts-1-chromium-linux.png new file mode 100644 index 0000000000..2fc9d8e70a Binary files /dev/null and b/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-loads-the-budget-page-with-budgeted-amounts-1-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-opens-individual-account-page-and-checks-that-filtering-is-working-1-chromium-linux.png b/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-opens-individual-account-page-and-checks-that-filtering-is-working-1-chromium-linux.png new file mode 100644 index 0000000000..06a183b0da Binary files /dev/null and b/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-opens-individual-account-page-and-checks-that-filtering-is-working-1-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-opens-individual-account-page-and-checks-that-filtering-is-working-2-chromium-linux.png b/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-opens-individual-account-page-and-checks-that-filtering-is-working-2-chromium-linux.png new file mode 100644 index 0000000000..061d910343 Binary files /dev/null and b/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-opens-individual-account-page-and-checks-that-filtering-is-working-2-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-opens-individual-account-page-and-checks-that-filtering-is-working-3-chromium-linux.png b/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-opens-individual-account-page-and-checks-that-filtering-is-working-3-chromium-linux.png new file mode 100644 index 0000000000..d239e73fe1 Binary files /dev/null and b/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-opens-individual-account-page-and-checks-that-filtering-is-working-3-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-opens-the-accounts-page-and-asserts-on-balances-1-chromium-linux.png b/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-opens-the-accounts-page-and-asserts-on-balances-1-chromium-linux.png new file mode 100644 index 0000000000..207a3d5344 Binary files /dev/null and b/packages/desktop-client/e2e/mobile.test.js-snapshots/Mobile-opens-the-accounts-page-and-asserts-on-balances-1-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/onboarding.test.js b/packages/desktop-client/e2e/onboarding.test.js index db512c42ff..c748ad9352 100644 --- a/packages/desktop-client/e2e/onboarding.test.js +++ b/packages/desktop-client/e2e/onboarding.test.js @@ -5,6 +5,7 @@ import { test, expect } from '@playwright/test'; import { AccountPage } from './page-models/account-page'; import { ConfigurationPage } from './page-models/configuration-page'; import { Navigation } from './page-models/navigation'; +import screenshotConfig from './screenshot.config'; test.describe('Onboarding', () => { let page; @@ -23,6 +24,14 @@ test.describe('Onboarding', () => { await page.close(); }); + test('checks the page visuals', async () => { + await expect(configurationPage.heading).toHaveText('Where’s the server?'); + await expect(page).toHaveScreenshot(screenshotConfig(page)); + + await configurationPage.clickOnNoServer(); + await expect(page).toHaveScreenshot(screenshotConfig(page)); + }); + test('creates a new budget file by importing YNAB4 budget', async () => { await configurationPage.clickOnNoServer(); const budgetPage = await configurationPage.importBudget( @@ -64,7 +73,7 @@ test.describe('Onboarding', () => { path.resolve(__dirname, 'data/actual-demo-budget.zip'), ); - await expect(budgetPage.budgetTable).toBeVisible(); + await expect(budgetPage.budgetTable).toBeVisible({ timeout: 20_000 }); const accountPage = await navigation.goToAccountPage('Ally Savings'); await expect(accountPage.accountBalance).toHaveText('1,772.80'); diff --git a/packages/desktop-client/e2e/onboarding.test.js-snapshots/Onboarding-checks-the-page-visuals-1-chromium-linux.png b/packages/desktop-client/e2e/onboarding.test.js-snapshots/Onboarding-checks-the-page-visuals-1-chromium-linux.png new file mode 100644 index 0000000000..2ff0f44b32 Binary files /dev/null and b/packages/desktop-client/e2e/onboarding.test.js-snapshots/Onboarding-checks-the-page-visuals-1-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/onboarding.test.js-snapshots/Onboarding-checks-the-page-visuals-2-chromium-linux.png b/packages/desktop-client/e2e/onboarding.test.js-snapshots/Onboarding-checks-the-page-visuals-2-chromium-linux.png new file mode 100644 index 0000000000..b37b61f98d Binary files /dev/null and b/packages/desktop-client/e2e/onboarding.test.js-snapshots/Onboarding-checks-the-page-visuals-2-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/page-models/account-page.js b/packages/desktop-client/e2e/page-models/account-page.js index a4c98e8132..9f0127bd07 100644 --- a/packages/desktop-client/e2e/page-models/account-page.js +++ b/packages/desktop-client/e2e/page-models/account-page.js @@ -70,15 +70,15 @@ export class AccountPage { * Retrieve the data for the nth-transaction. * 0-based index */ - async getNthTransaction(index) { + getNthTransaction(index) { const row = this.transactionTableRow.nth(index); return { - payee: await row.getByTestId('payee').textContent(), - notes: await row.getByTestId('notes').textContent(), - category: await row.getByTestId('category').textContent(), - debit: await row.getByTestId('debit').textContent(), - credit: await row.getByTestId('credit').textContent(), + payee: row.getByTestId('payee'), + notes: row.getByTestId('notes'), + category: row.getByTestId('category'), + debit: row.getByTestId('debit'), + credit: row.getByTestId('credit'), }; } diff --git a/packages/desktop-client/e2e/page-models/mobile-accounts-page.js b/packages/desktop-client/e2e/page-models/mobile-accounts-page.js index 5c0b8c601c..f91bb147fd 100644 --- a/packages/desktop-client/e2e/page-models/mobile-accounts-page.js +++ b/packages/desktop-client/e2e/page-models/mobile-accounts-page.js @@ -14,11 +14,8 @@ export class MobileAccountsPage { const accountRow = this.accounts.nth(idx); return { - name: await accountRow.getByTestId('account-name').textContent(), - balance: parseInt( - await accountRow.getByTestId('account-balance').textContent(), - 10, - ), + name: accountRow.getByTestId('account-name'), + balance: accountRow.getByTestId('account-balance'), }; } diff --git a/packages/desktop-client/e2e/reports.test.js b/packages/desktop-client/e2e/reports.test.js index 554b203d45..6a0763223b 100644 --- a/packages/desktop-client/e2e/reports.test.js +++ b/packages/desktop-client/e2e/reports.test.js @@ -2,6 +2,7 @@ import { test, expect } from '@playwright/test'; import { ConfigurationPage } from './page-models/configuration-page'; import { Navigation } from './page-models/navigation'; +import screenshotConfig from './screenshot.config'; test.describe('Reports', () => { let page; @@ -31,5 +32,6 @@ test.describe('Reports', () => { const reports = await reportsPage.getAvailableReportList(); expect(reports).toEqual(['Net Worth', 'Cash Flow']); + await expect(page).toHaveScreenshot(screenshotConfig(page)); }); }); diff --git a/packages/desktop-client/e2e/reports.test.js-snapshots/Reports-loads-net-worth-and-cash-flow-reports-1-chromium-linux.png b/packages/desktop-client/e2e/reports.test.js-snapshots/Reports-loads-net-worth-and-cash-flow-reports-1-chromium-linux.png new file mode 100644 index 0000000000..d8d4733a45 Binary files /dev/null and b/packages/desktop-client/e2e/reports.test.js-snapshots/Reports-loads-net-worth-and-cash-flow-reports-1-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/rules.test.js b/packages/desktop-client/e2e/rules.test.js index bdedea7cd0..d379d6d3fe 100644 --- a/packages/desktop-client/e2e/rules.test.js +++ b/packages/desktop-client/e2e/rules.test.js @@ -2,6 +2,7 @@ import { test, expect } from '@playwright/test'; import { ConfigurationPage } from './page-models/configuration-page'; import { Navigation } from './page-models/navigation'; +import screenshotConfig from './screenshot.config'; test.describe('Rules', () => { let page; @@ -26,6 +27,10 @@ test.describe('Rules', () => { rulesPage = await navigation.goToRulesPage(); }); + test('checks the page visuals', async () => { + await expect(page).toHaveScreenshot(screenshotConfig(page)); + }); + test('creates a rule and makes sure it is applied when creating a transaction', async () => { await rulesPage.createRule({ conditions: [ @@ -47,6 +52,7 @@ test.describe('Rules', () => { conditions: ['payee is Fast Internet'], actions: ['set category to General'], }); + await expect(page).toHaveScreenshot(screenshotConfig(page)); const accountPage = await navigation.goToAccountPage('Bank of America'); @@ -55,10 +61,10 @@ test.describe('Rules', () => { debit: '12.34', }); - expect(await accountPage.getNthTransaction(0)).toMatchObject({ - payee: 'Fast Internet', - category: 'General', - debit: '12.34', - }); + const transaction = accountPage.getNthTransaction(0); + await expect(transaction.payee).toHaveText('Fast Internet'); + await expect(transaction.category).toHaveText('General'); + await expect(transaction.debit).toHaveText('12.34'); + await expect(page).toHaveScreenshot(screenshotConfig(page)); }); }); diff --git a/packages/desktop-client/e2e/rules.test.js-snapshots/Rules-checks-the-page-visuals-1-chromium-linux.png b/packages/desktop-client/e2e/rules.test.js-snapshots/Rules-checks-the-page-visuals-1-chromium-linux.png new file mode 100644 index 0000000000..48bd21d4b6 Binary files /dev/null and b/packages/desktop-client/e2e/rules.test.js-snapshots/Rules-checks-the-page-visuals-1-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/rules.test.js-snapshots/Rules-creates-a-rule-and-makes-sure-it-is-applied-when-creating-a-transaction-1-chromium-linux.png b/packages/desktop-client/e2e/rules.test.js-snapshots/Rules-creates-a-rule-and-makes-sure-it-is-applied-when-creating-a-transaction-1-chromium-linux.png new file mode 100644 index 0000000000..551c1cc27d Binary files /dev/null and b/packages/desktop-client/e2e/rules.test.js-snapshots/Rules-creates-a-rule-and-makes-sure-it-is-applied-when-creating-a-transaction-1-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/rules.test.js-snapshots/Rules-creates-a-rule-and-makes-sure-it-is-applied-when-creating-a-transaction-2-chromium-linux.png b/packages/desktop-client/e2e/rules.test.js-snapshots/Rules-creates-a-rule-and-makes-sure-it-is-applied-when-creating-a-transaction-2-chromium-linux.png new file mode 100644 index 0000000000..7fe56db1ab Binary files /dev/null and b/packages/desktop-client/e2e/rules.test.js-snapshots/Rules-creates-a-rule-and-makes-sure-it-is-applied-when-creating-a-transaction-2-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/schedules.test.js b/packages/desktop-client/e2e/schedules.test.js index 6b79b95f83..683ddf9b2a 100644 --- a/packages/desktop-client/e2e/schedules.test.js +++ b/packages/desktop-client/e2e/schedules.test.js @@ -2,6 +2,7 @@ import { test, expect } from '@playwright/test'; import { ConfigurationPage } from './page-models/configuration-page'; import { Navigation } from './page-models/navigation'; +import screenshotConfig from './screenshot.config'; test.describe('Schedules', () => { let page; @@ -26,6 +27,10 @@ test.describe('Schedules', () => { schedulesPage = await navigation.goToSchedulesPage(); }); + test('checks the page visuals', async () => { + await expect(page).toHaveScreenshot(screenshotConfig(page)); + }); + test('creates a new schedule, posts the transaction and later completes it', async () => { await schedulesPage.addNewSchedule({ payee: 'Home Depot', @@ -39,20 +44,21 @@ test.describe('Schedules', () => { amount: '~25.00', status: 'Due', }); + await expect(page).toHaveScreenshot(screenshotConfig(page)); await schedulesPage.postNthSchedule(0); expect(await schedulesPage.getNthSchedule(0)).toMatchObject({ status: 'Paid', }); + await expect(page).toHaveScreenshot(screenshotConfig(page)); // Go to transactions page const accountPage = await navigation.goToAccountPage('HSBC'); - expect(await accountPage.getNthTransaction(0)).toMatchObject({ - payee: 'Home Depot', - category: 'Categorize', - debit: '25.00', - credit: '', - }); + const transaction = accountPage.getNthTransaction(0); + await expect(transaction.payee).toHaveText('Home Depot'); + await expect(transaction.category).toHaveText('Categorize'); + await expect(transaction.debit).toHaveText('25.00'); + await expect(transaction.credit).toHaveText(''); // go to rules page const rulesPage = await navigation.goToRulesPage(); diff --git a/packages/desktop-client/e2e/schedules.test.js-snapshots/Schedules-checks-the-page-visuals-1-chromium-linux.png b/packages/desktop-client/e2e/schedules.test.js-snapshots/Schedules-checks-the-page-visuals-1-chromium-linux.png new file mode 100644 index 0000000000..d671dfdf22 Binary files /dev/null and b/packages/desktop-client/e2e/schedules.test.js-snapshots/Schedules-checks-the-page-visuals-1-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/schedules.test.js-snapshots/Schedules-creates-a-new-schedule-posts-the-transaction-and-later-completes-it-1-chromium-linux.png b/packages/desktop-client/e2e/schedules.test.js-snapshots/Schedules-creates-a-new-schedule-posts-the-transaction-and-later-completes-it-1-chromium-linux.png new file mode 100644 index 0000000000..6ce00637e0 Binary files /dev/null and b/packages/desktop-client/e2e/schedules.test.js-snapshots/Schedules-creates-a-new-schedule-posts-the-transaction-and-later-completes-it-1-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/schedules.test.js-snapshots/Schedules-creates-a-new-schedule-posts-the-transaction-and-later-completes-it-2-chromium-linux.png b/packages/desktop-client/e2e/schedules.test.js-snapshots/Schedules-creates-a-new-schedule-posts-the-transaction-and-later-completes-it-2-chromium-linux.png new file mode 100644 index 0000000000..ef78d4a770 Binary files /dev/null and b/packages/desktop-client/e2e/schedules.test.js-snapshots/Schedules-creates-a-new-schedule-posts-the-transaction-and-later-completes-it-2-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/screenshot.config.js b/packages/desktop-client/e2e/screenshot.config.js new file mode 100644 index 0000000000..2debabf972 --- /dev/null +++ b/packages/desktop-client/e2e/screenshot.config.js @@ -0,0 +1,6 @@ +export default function screenshotConfig(page) { + return { + // eslint-disable-next-line rulesdir/typography + mask: [page.locator('[data-vrt-mask="true"]')], + }; +} diff --git a/packages/desktop-client/e2e/settings.test.js b/packages/desktop-client/e2e/settings.test.js index 058f9a8616..0bcf3c00ab 100644 --- a/packages/desktop-client/e2e/settings.test.js +++ b/packages/desktop-client/e2e/settings.test.js @@ -2,6 +2,7 @@ import { test, expect } from '@playwright/test'; import { ConfigurationPage } from './page-models/configuration-page'; import { Navigation } from './page-models/navigation'; +import screenshotConfig from './screenshot.config'; test.describe('Settings', () => { let page; @@ -26,6 +27,10 @@ test.describe('Settings', () => { settingsPage = await navigation.goToSettingsPage(); }); + test('checks the page visuals', async () => { + await expect(page).toHaveScreenshot(screenshotConfig(page)); + }); + test('downloads the export of the budget', async () => { const downloadPromise = page.waitForEvent('download'); diff --git a/packages/desktop-client/e2e/settings.test.js-snapshots/Settings-checks-the-page-visuals-1-chromium-linux.png b/packages/desktop-client/e2e/settings.test.js-snapshots/Settings-checks-the-page-visuals-1-chromium-linux.png new file mode 100644 index 0000000000..0beaee67da Binary files /dev/null and b/packages/desktop-client/e2e/settings.test.js-snapshots/Settings-checks-the-page-visuals-1-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/transactions.test.js b/packages/desktop-client/e2e/transactions.test.js index ed6be9e058..a9e4975feb 100644 --- a/packages/desktop-client/e2e/transactions.test.js +++ b/packages/desktop-client/e2e/transactions.test.js @@ -2,6 +2,7 @@ import { test, expect } from '@playwright/test'; import { ConfigurationPage } from './page-models/configuration-page'; import { Navigation } from './page-models/navigation'; +import screenshotConfig from './screenshot.config'; test.describe('Transactions', () => { let page; @@ -26,6 +27,10 @@ test.describe('Transactions', () => { accountPage = await navigation.goToAccountPage('Ally Savings'); }); + test('checks the page visuals', async () => { + await expect(page).toHaveScreenshot(screenshotConfig(page)); + }); + test('creates a test transaction', async () => { await accountPage.createSingleTransaction({ payee: 'Home Depot', @@ -34,13 +39,13 @@ test.describe('Transactions', () => { debit: '12.34', }); - expect(await accountPage.getNthTransaction(0)).toMatchObject({ - payee: 'Home Depot', - notes: 'Notes field', - category: 'Food', - debit: '12.34', - credit: '', - }); + const transaction = accountPage.getNthTransaction(0); + await expect(transaction.payee).toHaveText('Home Depot'); + await expect(transaction.notes).toHaveText('Notes field'); + await expect(transaction.category).toHaveText('Food'); + await expect(transaction.debit).toHaveText('12.34'); + await expect(transaction.credit).toHaveText(''); + await expect(page).toHaveScreenshot(screenshotConfig(page)); }); test('creates a split test transaction', async () => { @@ -59,26 +64,26 @@ test.describe('Transactions', () => { }, ]); - expect(await accountPage.getNthTransaction(0)).toMatchObject({ - payee: 'Krogger', - notes: 'Notes', - category: 'Split', - debit: '333.33', - credit: '', - }); - expect(await accountPage.getNthTransaction(1)).toMatchObject({ - payee: 'Krogger', - notes: '', - category: 'General', - debit: '222.22', - credit: '', - }); - expect(await accountPage.getNthTransaction(2)).toMatchObject({ - payee: 'Krogger', - notes: '', - category: 'Categorize', - debit: '111.11', - credit: '', - }); + const firstTransaction = accountPage.getNthTransaction(0); + await expect(firstTransaction.payee).toHaveText('Krogger'); + await expect(firstTransaction.notes).toHaveText('Notes'); + await expect(firstTransaction.category).toHaveText('Split'); + await expect(firstTransaction.debit).toHaveText('333.33'); + await expect(firstTransaction.credit).toHaveText(''); + + const secondTransaction = accountPage.getNthTransaction(1); + await expect(secondTransaction.payee).toHaveText('Krogger'); + await expect(secondTransaction.notes).toHaveText(''); + await expect(secondTransaction.category).toHaveText('General'); + await expect(secondTransaction.debit).toHaveText('222.22'); + await expect(secondTransaction.credit).toHaveText(''); + + const thirdTransaction = accountPage.getNthTransaction(2); + await expect(thirdTransaction.payee).toHaveText('Krogger'); + await expect(thirdTransaction.notes).toHaveText(''); + await expect(thirdTransaction.category).toHaveText('Categorize'); + await expect(thirdTransaction.debit).toHaveText('111.11'); + await expect(thirdTransaction.credit).toHaveText(''); + await expect(page).toHaveScreenshot(screenshotConfig(page)); }); }); diff --git a/packages/desktop-client/e2e/transactions.test.js-snapshots/Transactions-checks-the-page-visuals-1-chromium-linux.png b/packages/desktop-client/e2e/transactions.test.js-snapshots/Transactions-checks-the-page-visuals-1-chromium-linux.png new file mode 100644 index 0000000000..fa44cbe525 Binary files /dev/null and b/packages/desktop-client/e2e/transactions.test.js-snapshots/Transactions-checks-the-page-visuals-1-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/transactions.test.js-snapshots/Transactions-creates-a-split-test-transaction-1-chromium-linux.png b/packages/desktop-client/e2e/transactions.test.js-snapshots/Transactions-creates-a-split-test-transaction-1-chromium-linux.png new file mode 100644 index 0000000000..ad87197944 Binary files /dev/null and b/packages/desktop-client/e2e/transactions.test.js-snapshots/Transactions-creates-a-split-test-transaction-1-chromium-linux.png differ diff --git a/packages/desktop-client/e2e/transactions.test.js-snapshots/Transactions-creates-a-test-transaction-1-chromium-linux.png b/packages/desktop-client/e2e/transactions.test.js-snapshots/Transactions-creates-a-test-transaction-1-chromium-linux.png new file mode 100644 index 0000000000..b4e9668975 Binary files /dev/null and b/packages/desktop-client/e2e/transactions.test.js-snapshots/Transactions-creates-a-test-transaction-1-chromium-linux.png differ diff --git a/packages/desktop-client/package.json b/packages/desktop-client/package.json index 4221044e30..6c082c4cf0 100644 --- a/packages/desktop-client/package.json +++ b/packages/desktop-client/package.json @@ -7,7 +7,7 @@ ], "devDependencies": { "@juggle/resize-observer": "^3.1.2", - "@playwright/test": "^1.29.1", + "@playwright/test": "^1.37.1", "@reach/listbox": "^0.18.0", "@react-aria/focus": "^3.8.0", "@react-aria/listbox": "^3.6.1", @@ -66,7 +66,8 @@ "build:browser": "cross-env ./bin/build-browser", "generate:icons": "rm src/icons/*/*.js; cd src/icons && svgr --expand-props start --ext js -d . .", "test": "react-app-rewired test", - "e2e": "npx playwright test --browser=chromium" + "e2e": "npx playwright test --browser=chromium", + "vrt": "cross-env VRT=true npx playwright test --browser=chromium" }, "jest": { "setupFilesAfterEnv": [ diff --git a/packages/desktop-client/playwright.config.js b/packages/desktop-client/playwright.config.js index 1969400a2f..b3c1455417 100644 --- a/packages/desktop-client/playwright.config.js +++ b/packages/desktop-client/playwright.config.js @@ -1,11 +1,39 @@ -module.exports = { +import { expect, defineConfig } from '@playwright/test'; + +// Disable screenshot assertions in regular e2e tests; +// only enable them when doing VRT tests +if (!process.env.VRT) { + expect.extend({ + toHaveScreenshot() { + return { + message: () => 'passed', + pass: true, + }; + }, + }); +} + +// eslint-disable-next-line import/no-unused-modules +export default defineConfig({ timeout: 20000, // 20 seconds retries: 1, testDir: 'e2e/', use: { + userAgent: 'playwright', screenshot: 'on', browserName: 'chromium', baseURL: process.env.E2E_START_URL ?? 'http://localhost:3001', trace: 'on-first-retry', + ignoreHTTPSErrors: true, }, -}; + expect: { + toHaveScreenshot: { + // Approx. 2% difference (1000 pixels) + maxDiffPixels: 1000, + }, + toMatchSnapshot: { + // 2% difference is acceptable + maxDiffPixelRatio: 0.02, + }, + }, +}); diff --git a/packages/desktop-client/src/components/App.tsx b/packages/desktop-client/src/components/App.tsx index 32cc2d7440..bdfdd3841a 100644 --- a/packages/desktop-client/src/components/App.tsx +++ b/packages/desktop-client/src/components/App.tsx @@ -6,6 +6,7 @@ import { } from 'react-error-boundary'; import { useSelector } from 'react-redux'; +import * as Platform from 'loot-core/src/client/platform'; import { init as initConnection, send, @@ -174,7 +175,9 @@ function AppWrapper() { }} > - {process.env.REACT_APP_REVIEW_ID && } + {process.env.REACT_APP_REVIEW_ID && !Platform.isPlaywright && ( + + )} @@ -196,7 +197,11 @@ class AccountList extends Component { /> ))} - + {offbudgetAccounts.map((acct, idx) => ( {/* eslint-disable-next-line rulesdir/typography */} {monthUtils.format(currentMonth, "MMMM ''yy")} diff --git a/packages/desktop-client/src/components/budget/MonthPicker.js b/packages/desktop-client/src/components/budget/MonthPicker.js index c9520fadbb..a3bd54791f 100644 --- a/packages/desktop-client/src/components/budget/MonthPicker.js +++ b/packages/desktop-client/src/components/budget/MonthPicker.js @@ -1,5 +1,6 @@ import { useState } from 'react'; +import * as Platform from 'loot-core/src/client/platform'; import * as monthUtils from 'loot-core/src/shared/months'; import useResizeObserver from '../../hooks/useResizeObserver'; @@ -140,9 +141,10 @@ export const MonthPicker = ({ onClick={() => onSelect(month)} onMouseEnter={() => setHoverId(idx)} onMouseLeave={() => setHoverId(null)} + data-vrt-mask > {size === 'small' ? monthName[0] : monthName} - {showYearHeader && ( + {!Platform.isPlaywright && showYearHeader && ( {monthUtils.format(month, 'MMMM')} diff --git a/packages/desktop-client/src/components/budget/rollover/BudgetSummary.tsx b/packages/desktop-client/src/components/budget/rollover/BudgetSummary.tsx index c67b4fa8eb..9bbe6fe9db 100644 --- a/packages/desktop-client/src/components/budget/rollover/BudgetSummary.tsx +++ b/packages/desktop-client/src/components/budget/rollover/BudgetSummary.tsx @@ -134,7 +134,7 @@ function TotalsList({ prevMonthName, collapsed }: TotalsListProps) { Available Funds - Overspent in {prevMonthName} + Overspent in {prevMonthName} Budgeted For Next Month @@ -347,6 +347,7 @@ export function BudgetSummary({ }, currentMonth === month && { fontWeight: 'bold' }, ])} + data-vrt-mask > {monthUtils.format(month, 'MMMM')} diff --git a/packages/desktop-client/src/components/common/View.tsx b/packages/desktop-client/src/components/common/View.tsx index 4133c2d352..72fea498e8 100644 --- a/packages/desktop-client/src/components/common/View.tsx +++ b/packages/desktop-client/src/components/common/View.tsx @@ -16,13 +16,13 @@ const View = (props: ViewProps) => { // everywhere and we can avoid any perf penalty that glamor would // incur. - const { style, nativeStyle, innerRef, ...restProps } = props; + const { style, nativeStyle, innerRef, className = '', ...restProps } = props; return (
); }; diff --git a/packages/desktop-client/src/components/manager/ConfigServer.tsx b/packages/desktop-client/src/components/manager/ConfigServer.tsx index b12d4790ee..7dbda77584 100644 --- a/packages/desktop-client/src/components/manager/ConfigServer.tsx +++ b/packages/desktop-client/src/components/manager/ConfigServer.tsx @@ -182,7 +182,7 @@ export default function ConfigServer() { }} onClick={onSameDomain} > - Use {window.location.origin.replace(/https?:\/\//, '')} + Use current domain )}