Allow libofx to handle decoding imported files (#591)

* Allow libofx to handle decoding imported files

* Add releve.qfx as a test file

* Remove irrelevant redacted transactions from test file

* Fix console overload from long console.warn stacks

packages/loot-core/src/mocks/files/8859-1.qfx

@@ -0,0 +1,63 @@
+OFXHEADER:100
+DATA:OFXSGML
+VERSION:102
+SECURITY:TYPE1
+ENCODING:USASCII
+CHARSET:8859-1
+COMPRESSION:NONE
+OLDFILEUID:NONE
+NEWFILEUID:NONE
+<OFX>
+<SIGNONMSGSRSV1>
+<SONRS>
+<STATUS>
+<CODE>0
+<SEVERITY>INFO
+<MESSAGE>OK
+</STATUS>
+<DTSERVER>20221020231556
+<USERKEY>3AAAAAAAAAAAAAA
+<INTU.BID>00999
+<LANGUAGE>FRA
+</SONRS>
+</SIGNONMSGSRSV1>
+<BANKMSGSRSV1>
+<STMTTRNRS>
+<TRNUID>AAAA-2022102023155614357
+<STATUS>
+<CODE>0
+<SEVERITY>INFO
+<MESSAGE>OK
+</STATUS>
+<STMTRS>
+<CURDEF>CAD
+<BANKACCTFROM>
+<BANKID>700012345
+<BRANCHID>0055666
+<ACCTID>999-12345-0055666-EOP
+<ACCTTYPE>CHECKING
+</BANKACCTFROM>
+<BANKTRANLIST>
+<DTSTART>20221019120000
+<DTEND>20221019120000
+<STMTTRN>
+<TRNTYPE>DEBIT
+<DTPOSTED>20221019120000
+<TRNAMT>-20.00
+<FITID>wSoKuCS77
+<NAME>Paiement facture/Carte prépayée
+<MEMO>PWW
+</STMTTRN>
+</BANKTRANLIST>
+<LEDGERBAL>
+<BALAMT>9999.99
+<DTASOF>20221020231556
+</LEDGERBAL>
+<AVAILBAL>
+<BALAMT>9999.99
+<DTASOF>20221020231556
+</AVAILBAL>
+</STMTRS>
+</STMTTRNRS>
+</BANKMSGSRSV1>
+</OFX>

packages/loot-core/src/server/accounts/__snapshots__/parse-file.test.js.snap

@@ -1,5 +1,34 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`File import handles non-ASCII characters 1`] = `
+Array [
+  Object {
+    "acct": "one",
+    "amount": -2000,
+    "category": null,
+    "cleared": 1,
+    "date": 20221019,
+    "description": "id2",
+    "error": null,
+    "financial_id": "wSoKuCS77",
+    "id": "id3",
+    "imported_description": "Paiement facture/Carte prépayée",
+    "isChild": 0,
+    "isParent": 0,
+    "location": null,
+    "notes": "PWW",
+    "parent_id": null,
+    "pending": 0,
+    "schedule": null,
+    "sort_order": 123456789,
+    "starting_balance_flag": 0,
+    "tombstone": 0,
+    "transferred_id": null,
+    "type": null,
+  },
+]
+`;
+
 exports[`File import ofx import works 1`] = `
 Array [
   Object {

packages/loot-core/src/server/accounts/parse-file.js

@@ -91,7 +91,7 @@ async function parseOFX(filepath) {
   await initModule();
 
   let errors = [];
-  let contents = await fs.readFile(filepath);
+  let contents = await fs.readFile(filepath, 'binary');
 
   let data;
   try {

packages/loot-core/src/server/accounts/parse-file.test.js

@@ -9,6 +9,17 @@ import { reconcileTransactions } from './sync';
 
 beforeEach(global.emptyDatabase());
 
+// libofx spits out errors that contain the entire
+// source code of the file in the stack which makes
+// it hard to test.
+let old = console.warn;
+beforeAll(() => {
+  console.warn = () => {};
+});
+afterAll(() => {
+  console.warn = old;
+});
+
 async function getTransactions(accountId) {
   return db.runQuery(
     'SELECT * FROM transactions WHERE acct = ?',
@@ -99,4 +110,17 @@ describe('File import', () => {
     expect(res.errors.length).toBe(1);
     expect(res.errors[0].message).toBe('Invalid file type');
   }, 45000);
+
+  test('handles non-ASCII characters', async () => {
+    prefs.loadPrefs();
+    await db.insertAccount({ id: 'one', name: 'one' });
+
+    let { errors } = await importFileWithRealTime(
+      'one',
+      __dirname + '/../../mocks/files/8859-1.qfx',
+      'yyyy-MM-dd'
+    );
+    expect(errors.length).toBe(0);
+    expect(await getTransactions('one')).toMatchSnapshot();
+  });
 });

packages/node-libofx/ffi.js

@@ -3,7 +3,7 @@ function create(libofx) {
     init: libofx.cwrap('init', null, ['number']),
     debug: libofx.cwrap('debug', null, []),
     get_new_context: libofx.cwrap('get_new_context', 'number', []),
-    parse_data: libofx.cwrap('parse_data', null, ['number', 'string']),
+    parse_data: libofx.cwrap('parse_data', null, ['number', 'array']),
 
     ofx_set_transaction_cb: libofx.cwrap('ofx_set_transaction_cb', null, [
       'number',