github-starred/actual
2
0
Fork 0
mirror of https://github.com/actualbudget/actual.git synced 2026-04-27 17:48:17 -05:00
Code Issues 1.3k Packages Projects Releases 54 Wiki Activity

✨ ability to add migrations (#267)

Browse Source
This commit is contained in:
Matiss Janis Aboltins
2023-11-07 18:10:16 +00:00
committed by GitHub
parent 494d67459f
commit 1f79708ea3
22 changed files with 229 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -15,6 +15,8 @@ build/
*.pem
*.key
artifacts.json
.migrate
.migrate-test
# Yarn
.pnp.*

11
app.js
View File

@@ -1,6 +1,9 @@
import run from './src/app.js';
import runMigrations from './src/migrations.js';
run().catch((err) => {
console.log('Error starting app:', err);
process.exit(1);
});
runMigrations()
.then(run)
.catch((err) => {
console.log('Error starting app:', err);
process.exit(1);
});

3
jest.config.json
View File

@@ -1,5 +1,6 @@
{
"setupFiles": ["./jest.setup.js"],
"globalSetup": "./jest.global-setup.js",
"globalTeardown": "./jest.global-teardown.js",
"testPathIgnorePatterns": ["dist", "/node_modules/", "/build/"],
"roots": ["<rootDir>"],
"moduleFileExtensions": ["ts", "js", "json"],

10
jest.global-setup.js Normal file
View File

@@ -0,0 +1,10 @@
import getAccountDb from './src/account-db.js';
import runMigrations from './src/migrations.js';
export default async function setup() {
await runMigrations();
// Insert a fake "valid-token" fixture that can be reused
const db = getAccountDb();
await db.mutate('INSERT INTO sessions (token) VALUES (?)', ['valid-token']);
}

5
jest.global-teardown.js Normal file
View File

@@ -0,0 +1,5 @@
import runMigrations from './src/migrations.js';
export default async function teardown() {
await runMigrations('down');
}

17
jest.setup.js
View File

@@ -1,17 +0,0 @@
import fs from 'node:fs';
import path from 'node:path';
import getAccountDb from './src/account-db.js';
import config from './src/load-config.js';
// Delete previous test database (force creation of a new one)
const dbPath = path.join(config.serverFiles, 'account.sqlite');
if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath);
// Create path for test user files and delete previous files there
if (fs.existsSync(config.userFiles))
fs.rmSync(config.userFiles, { recursive: true });
fs.mkdirSync(config.userFiles);
// Insert a fake "valid-token" fixture that can be reused
const db = getAccountDb();
db.mutate('INSERT INTO sessions (token) VALUES (?)', ['valid-token']);

24
migrations/1694360000000-create-folders.js Normal file
View File

@@ -0,0 +1,24 @@
import fs from 'node:fs/promises';
import config from '../src/load-config.js';
async function ensureExists(path) {
try {
await fs.mkdir(path);
} catch (err) {
if (err.code == 'EEXIST') {
return null;
}
throw err;
}
}
export const up = async function () {
await ensureExists(config.serverFiles);
await ensureExists(config.userFiles);
};
export const down = async function () {
await fs.rm(config.serverFiles, { recursive: true, force: true });
await fs.rm(config.userFiles, { recursive: true, force: true });
};

30
migrations/1694360479680-create-account-db.js Normal file
View File

@@ -0,0 +1,30 @@
import getAccountDb from '../src/account-db.js';
export const up = async function () {
await getAccountDb().exec(`
CREATE TABLE IF NOT EXISTS auth
(password TEXT PRIMARY KEY);
CREATE TABLE IF NOT EXISTS sessions
(token TEXT PRIMARY KEY);
CREATE TABLE IF NOT EXISTS files
(id TEXT PRIMARY KEY,
group_id TEXT,
sync_version SMALLINT,
encrypt_meta TEXT,
encrypt_keyid TEXT,
encrypt_salt TEXT,
encrypt_test TEXT,
deleted BOOLEAN DEFAULT FALSE,
name TEXT);
`);
};
export const down = async function () {
await getAccountDb().exec(`
DROP TABLE auth;
DROP TABLE sessions;
DROP TABLE files;
`);
};

16
migrations/1694362247011-create-secret-table.js Normal file
View File

@@ -0,0 +1,16 @@
import getAccountDb from '../src/account-db.js';
export const up = async function () {
await getAccountDb().exec(`
CREATE TABLE IF NOT EXISTS secrets (
name TEXT PRIMARY KEY,
value BLOB
);
`);
};
export const down = async function () {
await getAccountDb().exec(`
DROP TABLE secrets;
`);
};

1
package.json
View File

@@ -28,6 +28,7 @@
"express-rate-limit": "^6.7.0",
"express-response-size": "^0.0.3",
"jws": "^4.0.0",
"migrate": "^2.0.0",
"nordigen-node": "^1.2.6",
"uuid": "^9.0.0"
},

23
src/account-db.js
View File

@@ -1,34 +1,15 @@
import fs from 'node:fs';
import { join } from 'node:path';
import openDatabase from './db.js';
import config, { sqlDir } from './load-config.js';
import createDebug from 'debug';
import config from './load-config.js';
import * as uuid from 'uuid';
import * as bcrypt from 'bcrypt';
const debug = createDebug('actual:account-db');
let _accountDb = null;
export default function getAccountDb() {
if (_accountDb == null) {
if (!fs.existsSync(config.serverFiles)) {
debug(`creating server files directory: '${config.serverFiles}'`);
fs.mkdirSync(config.serverFiles);
}
let dbPath = join(config.serverFiles, 'account.sqlite');
let needsInit = !fs.existsSync(dbPath);
const dbPath = join(config.serverFiles, 'account.sqlite');
_accountDb = openDatabase(dbPath);
if (needsInit) {
debug(`initializing account database: '${dbPath}'`);
let initSql = fs.readFileSync(join(sqlDir, 'account.sql'), 'utf8');
_accountDb.exec(initSql);
} else {
debug(`opening account database: '${dbPath}'`);
}
}
return _accountDb;

4
src/app-account.js
View File

@@ -13,10 +13,6 @@ app.use(errorMiddleware);
export { app as handlers };
export function init() {
// eslint-disable-previous-line @typescript-eslint/no-empty-function
}
// Non-authenticated endpoints:
//
// /needs-bootstrap

43
src/app-gocardless/services/gocardless-service.js
View File

@@ -17,17 +17,12 @@ import jwt from 'jws';
import { SecretName, secretsService } from '../../services/secrets-service.js';
const GoCardlessClient = nordigenNode.default;
const goCardlessClient = new GoCardlessClient({
secretId: secretsService.get(SecretName.nordigen_secretId),
secretKey: secretsService.get(SecretName.nordigen_secretKey),
});
secretsService.onUpdate(SecretName.nordigen_secretId, (newSecret) => {
goCardlessClient.secretId = newSecret;
});
secretsService.onUpdate(SecretName.nordigen_secretKey, (newSecret) => {
goCardlessClient.secretKey = newSecret;
});
const getGocardlessClient = () =>
new GoCardlessClient({
secretId: secretsService.get(SecretName.nordigen_secretId),
secretKey: secretsService.get(SecretName.nordigen_secretKey),
});
export const handleGoCardlessError = (response) => {
switch (response.status_code) {
@@ -58,7 +53,9 @@ export const goCardlessService = {
* @returns {boolean}
*/
isConfigured: () => {
return !!(goCardlessClient.secretId && goCardlessClient.secretKey);
return !!(
getGocardlessClient().secretId && getGocardlessClient().secretKey
);
},
/**
@@ -76,7 +73,7 @@ export const goCardlessService = {
return clockTimestamp >= payload.exp;
};
if (isExpiredJwtToken(goCardlessClient.token)) {
if (isExpiredJwtToken(getGocardlessClient().token)) {
// Generate new access token. Token is valid for 24 hours
// Note: access_token is automatically injected to other requests after you successfully obtain it
const tokenData = await client.generateToken();
@@ -479,25 +476,25 @@ export const goCardlessService = {
*/
export const client = {
getBalances: async (accountId) =>
await goCardlessClient.account(accountId).getBalances(),
await getGocardlessClient().account(accountId).getBalances(),
getTransactions: async ({ accountId, dateFrom, dateTo }) =>
await goCardlessClient.account(accountId).getTransactions({
await getGocardlessClient().account(accountId).getTransactions({
dateFrom,
dateTo,
country: undefined,
}),
getInstitutions: async (country) =>
await goCardlessClient.institution.getInstitutions({ country }),
await getGocardlessClient().institution.getInstitutions({ country }),
getInstitutionById: async (institutionId) =>
await goCardlessClient.institution.getInstitutionById(institutionId),
await getGocardlessClient().institution.getInstitutionById(institutionId),
getDetails: async (accountId) =>
await goCardlessClient.account(accountId).getDetails(),
await getGocardlessClient().account(accountId).getDetails(),
getMetadata: async (accountId) =>
await goCardlessClient.account(accountId).getMetadata(),
await getGocardlessClient().account(accountId).getMetadata(),
getRequisitionById: async (requisitionId) =>
await goCardlessClient.requisition.getRequisitionById(requisitionId),
await getGocardlessClient().requisition.getRequisitionById(requisitionId),
deleteRequisition: async (requisitionId) =>
await goCardlessClient.requisition.deleteRequisition(requisitionId),
await getGocardlessClient().requisition.deleteRequisition(requisitionId),
initSession: async ({
redirectUrl,
institutionId,
@@ -509,7 +506,7 @@ export const client = {
redirectImmediate,
accountSelection,
}) =>
await goCardlessClient.initSession({
await getGocardlessClient().initSession({
redirectUrl,
institutionId,
referenceId,
@@ -520,7 +517,7 @@ export const client = {
redirectImmediate,
accountSelection,
}),
generateToken: async () => await goCardlessClient.generateToken(),
generateToken: async () => await getGocardlessClient().generateToken(),
exchangeToken: async ({ refreshToken }) =>
await goCardlessClient.exchangeToken({ refreshToken }),
await getGocardlessClient().exchangeToken({ refreshToken }),
};

3
src/app-sync.js
View File

@@ -15,9 +15,6 @@ const app = express();
app.use(errorMiddleware);
export { app as handlers };
// eslint-disable-next-line
export async function init() {}
// This is a version representing the internal format of sync
// messages. When this changes, all sync files need to be reset. We
// will check this version when syncing and notify the user if they

11
src/app.js
View File

@@ -71,17 +71,6 @@ function parseHTTPSConfig(value) {
}
export default async function run() {
if (!fs.existsSync(config.serverFiles)) {
fs.mkdirSync(config.serverFiles);
}
if (!fs.existsSync(config.userFiles)) {
fs.mkdirSync(config.userFiles);
}
await accountApp.init();
await syncApp.init();
if (config.https) {
const https = await import('node:https');
const httpsOptions = {

2
src/db.js
View File

@@ -27,7 +27,7 @@ class WrappedDatabase {
* @param {string} sql
*/
exec(sql) {
this.db.exec(sql);
return this.db.exec(sql);
}
/**

30
src/migrations.js Normal file
View File

@@ -0,0 +1,30 @@
import migrate from 'migrate';
import config from './load-config.js';
export default function run(direction = 'up') {
console.log(
`Checking if there are any migrations to run for direction "${direction}"...`,
);
return new Promise((resolve) =>
migrate.load(
{
stateStore: `.migrate${config.mode === 'test' ? '-test' : ''}`,
},
(err, set) => {
if (err) {
throw err;
}
set[direction]((err) => {
if (err) {
throw err;
}
console.log('Migrations: DONE');
resolve();
});
},
),
);
}

36
src/services/secrets-service.js
View File

@@ -1,7 +1,4 @@
import createDebug from 'debug';
import fs from 'node:fs';
import { sqlDir } from '../load-config.js';
import { join } from 'node:path';
import getAccountDb from '../account-db.js';
/**
@@ -18,18 +15,6 @@ class SecretsDb {
constructor() {
this.debug = createDebug('actual:secrets-db');
this.db = null;
this.initialize();
}
initialize() {
if (!this.db) {
this.db = this.open();
}
this.debug(`initializing secrets table'`);
//Create secret table if it doesn't exist
const initSql = fs.readFileSync(join(sqlDir, 'secrets.sql'), 'utf8');
this.db.exec(initSql);
}
open() {
@@ -64,7 +49,6 @@ class SecretsDb {
const secretsDb = new SecretsDb();
const _cachedSecrets = new Map();
const _observers = new Map();
/**
* A service for managing secrets stored in `secretsDb`.
*/
@@ -78,25 +62,6 @@ export const secretsService = {
return _cachedSecrets.get(name) ?? secretsDb.get(name)?.value ?? null;
},
/**
* Callbacks new secret value when a secret changes.
* @param {SecretName} name - The name of the secret to retrieve.
* @param {function(string): void} callback - The new secret value callback.
* @returns {void}
*/
onUpdate: (name, callback) => {
const observers = _observers.get(name) ?? [];
observers.push(callback);
_observers.set(name, observers);
},
_notifyObservers: (name, value) => {
const observers = _observers.get(name) ?? [];
for (const observer of observers) {
observer(value);
}
},
/**
* Sets the value of a secret by name.
* @param {SecretName} name - The name of the secret to set.
@@ -108,7 +73,6 @@ export const secretsService = {
if (result.changes === 1) {
_cachedSecrets.set(name, value);
secretsService._notifyObservers(name, value);
}
return result;
},

17
src/sql/account.sql
View File

@@ -1,17 +0,0 @@
CREATE TABLE auth
(password TEXT PRIMARY KEY);
CREATE TABLE sessions
(token TEXT PRIMARY KEY);
CREATE TABLE files
(id TEXT PRIMARY KEY,
group_id TEXT,
sync_version SMALLINT,
encrypt_meta TEXT,
encrypt_keyid TEXT,
encrypt_salt TEXT,
encrypt_test TEXT,
deleted BOOLEAN DEFAULT FALSE,
name TEXT);

4
src/sql/secrets.sql
View File

@@ -1,4 +0,0 @@
CREATE TABLE IF NOT EXISTS secrets (
name TEXT PRIMARY KEY,
value BLOB
);

6
upcoming-release-notes/267.md Normal file
View File

@@ -0,0 +1,6 @@
---
category: Enhancements
authors: [MatissJanis]
---
Ability to add and run database/fs migrations

74
yarn.lock
View File

@@ -1638,6 +1638,7 @@ __metadata:
express-response-size: ^0.0.3
jest: ^29.3.1
jws: ^4.0.0
migrate: ^2.0.0
nordigen-node: ^1.2.6
prettier: ^2.8.3
supertest: ^6.3.1
@@ -2129,7 +2130,7 @@ __metadata:
languageName: node
linkType: hard
"chalk@npm:^4.0.0":
"chalk@npm:^4.0.0, chalk@npm:^4.1.2":
version: 4.1.2
resolution: "chalk@npm:4.1.2"
dependencies:
@@ -2256,6 +2257,13 @@ __metadata:
languageName: node
linkType: hard
"commander@npm:^2.20.3":
version: 2.20.3
resolution: "commander@npm:2.20.3"
checksum: ab8c07884e42c3a8dbc5dd9592c606176c7eb5c1ca5ff274bcf907039b2c41de3626f684ea75ccf4d361ba004bbaff1f577d5384c155f3871e456bdf27becf9e
languageName: node
linkType: hard
"component-emitter@npm:^1.3.0":
version: 1.3.0
resolution: "component-emitter@npm:1.3.0"
@@ -2358,6 +2366,13 @@ __metadata:
languageName: node
linkType: hard
"dateformat@npm:^4.6.3":
version: 4.6.3
resolution: "dateformat@npm:4.6.3"
checksum: c3aa0617c0a5b30595122bc8d1bee6276a9221e4d392087b41cbbdf175d9662ae0e50d0d6dcdf45caeac5153c4b5b0844265f8cd2b2245451e3da19e39e3b65d
languageName: node
linkType: hard
"dayjs@npm:^1.11.3":
version: 1.11.7
resolution: "dayjs@npm:1.11.7"
@@ -2526,6 +2541,13 @@ __metadata:
languageName: node
linkType: hard
"dotenv@npm:^16.0.0":
version: 16.3.1
resolution: "dotenv@npm:16.3.1"
checksum: 15d75e7279018f4bafd0ee9706593dd14455ddb71b3bcba9c52574460b7ccaf67d5cf8b2c08a5af1a9da6db36c956a04a1192b101ee102a3e0cf8817bbcf3dfd
languageName: node
linkType: hard
"ecdsa-sig-formatter@npm:1.0.11":
version: 1.0.11
resolution: "ecdsa-sig-formatter@npm:1.0.11"
@@ -4447,6 +4469,29 @@ __metadata:
languageName: node
linkType: hard
"migrate@npm:^2.0.0":
version: 2.0.0
resolution: "migrate@npm:2.0.0"
dependencies:
chalk: ^4.1.2
commander: ^2.20.3
dateformat: ^4.6.3
dotenv: ^16.0.0
inherits: ^2.0.3
minimatch: ^9.0.1
mkdirp: ^3.0.1
slug: ^8.2.2
bin:
migrate: bin/migrate
migrate-create: bin/migrate-create
migrate-down: bin/migrate-down
migrate-init: bin/migrate-init
migrate-list: bin/migrate-list
migrate-up: bin/migrate-up
checksum: d7e5f476d32c638e7c6ee15e36e0a049f69ad3cb011011623658defa656cba2c75c0128d005ed0a06ea6d6426200297d809d1338db63cd580c03e8d42001a7bd
languageName: node
linkType: hard
"mime-db@npm:1.52.0":
version: 1.52.0
resolution: "mime-db@npm:1.52.0"
@@ -4513,6 +4558,15 @@ __metadata:
languageName: node
linkType: hard
"minimatch@npm:^9.0.1":
version: 9.0.3
resolution: "minimatch@npm:9.0.3"
dependencies:
brace-expansion: ^2.0.1
checksum: 253487976bf485b612f16bf57463520a14f512662e592e95c571afdab1442a6a6864b6c88f248ce6fc4ff0b6de04ac7aa6c8bb51e868e99d1d65eb0658a708b5
languageName: node
linkType: hard
"minimist@npm:^1.2.0, minimist@npm:^1.2.3":
version: 1.2.6
resolution: "minimist@npm:1.2.6"
@@ -4624,6 +4678,15 @@ __metadata:
languageName: node
linkType: hard
"mkdirp@npm:^3.0.1":
version: 3.0.1
resolution: "mkdirp@npm:3.0.1"
bin:
mkdirp: dist/cjs/src/bin.js
checksum: 972deb188e8fb55547f1e58d66bd6b4a3623bf0c7137802582602d73e6480c1c2268dcbafbfb1be466e00cc7e56ac514d7fd9334b7cf33e3e2ab547c16f83a8d
languageName: node
linkType: hard
"ms@npm:2.0.0":
version: 2.0.0
resolution: "ms@npm:2.0.0"
@@ -5503,6 +5566,15 @@ __metadata:
languageName: node
linkType: hard
"slug@npm:^8.2.2":
version: 8.2.3
resolution: "slug@npm:8.2.3"
bin:
slug: cli.js
checksum: eb2fbf8d13df0a94f09ffd7c20e02d5e88c1fdd51e178fe8e670937747dec5a97efd172956b914efd5eb3fadd65a306a68f4eed0327a8c5c9a8af30f2c95a46b
languageName: node
linkType: hard
"smart-buffer@npm:^4.2.0":
version: 4.2.0
resolution: "smart-buffer@npm:4.2.0"