Added configuration to CSV importer that allows to skip lines (#3234)

* Added configuration to CSV importer that allows to skip lines

* Fixed several type / parser check when import is executed with a different CSV structure on accounts with transactions

* Fixed linter warning

* Reverted changes on sync.ts as initial error is not occuring anymore.
This will also fix the tests again

---------

Co-authored-by: youngcw <calebyoung94@gmail.com>

packages/desktop-client/src/components/modals/ImportTransactions.jsx

@@ -186,20 +186,14 @@ function getInitialMappings(transactions) {
     return entry ? entry[0] : null;
   }
 
-  function isString(value) {
-    return typeof value === 'string' || value instanceof String;
-  }
-
   const dateField = key(
     fields.find(([name]) => name.toLowerCase().includes('date')) ||
-      fields.find(
-        ([, value]) => isString(value) && value.match(/^\d+[-/]\d+[-/]\d+$/),
-      ),
+      fields.find(([, value]) => String(value)?.match(/^\d+[-/]\d+[-/]\d+$/)),
   );
 
   const amountField = key(
     fields.find(([name]) => name.toLowerCase().includes('amount')) ||
-      fields.find(([, value]) => isString(value) && value.match(/^-?[.,\d]+$/)),
+      fields.find(([, value]) => String(value)?.match(/^-?[.,\d]+$/)),
   );
 
   const categoryField = key(
@@ -880,6 +874,9 @@ export function ImportTransactions({ options }) {
     prefs[`csv-delimiter-${accountId}`] ||
       (filename.endsWith('.tsv') ? '\t' : ','),
   );
+  const [skipLines, setSkipLines] = useState(
+    prefs[`csv-skip-lines-${accountId}`] ?? 0,
+  );
   const [hasHeaderRow, setHasHeaderRow] = useState(
     prefs[`csv-has-header-${accountId}`] ?? true,
   );
@@ -988,6 +985,7 @@ export function ImportTransactions({ options }) {
     const parseOptions = getParseOptions(fileType, {
       delimiter,
       hasHeaderRow,
+      skipLines,
       fallbackMissingPayeeToMemo,
     });
 
@@ -1040,6 +1038,7 @@ export function ImportTransactions({ options }) {
     const parseOptions = getParseOptions(fileType, {
       delimiter,
       hasHeaderRow,
+      skipLines,
       fallbackMissingPayeeToMemo,
     });
 
@@ -1196,6 +1195,8 @@ export function ImportTransactions({ options }) {
         [`csv-mappings-${accountId}`]: JSON.stringify(fieldMappings),
       });
       savePrefs({ [`csv-delimiter-${accountId}`]: delimiter });
+      savePrefs({ [`csv-has-header-${accountId}`]: hasHeaderRow });
+      savePrefs({ [`csv-skip-lines-${accountId}`]: skipLines });
     }
 
     if (filetype === 'csv' || filetype === 'qif') {
@@ -1282,6 +1283,10 @@ export function ImportTransactions({ options }) {
         );
         break;
       }
+      if (trans.payee == null || !(trans.payee instanceof String)) {
+        console.log(`Unable·to·parse·payee·${trans.payee || '(empty)'}`);
+        break;
+      }
 
       const { amount } = parseAmountFields(
         trans,
@@ -1575,6 +1580,34 @@ export function ImportTransactions({ options }) {
                             getParseOptions('csv', {
                               delimiter: value,
                               hasHeaderRow,
+                              skipLines,
+                            }),
+                          );
+                        }}
+                        style={{ width: 50 }}
+                      />
+                    </label>
+                    <label
+                      style={{
+                        display: 'flex',
+                        flexDirection: 'row',
+                        gap: 5,
+                        alignItems: 'baseline',
+                      }}
+                    >
+                      Skip lines:
+                      <Input
+                        type="number"
+                        value={skipLines}
+                        min="0"
+                        onChangeValue={value => {
+                          setSkipLines(+value);
+                          parse(
+                            filename,
+                            getParseOptions('csv', {
+                              delimiter,
+                              hasHeaderRow,
+                              skipLines: +value,
                             }),
                           );
                         }}
@@ -1591,6 +1624,7 @@ export function ImportTransactions({ options }) {
                           getParseOptions('csv', {
                             delimiter,
                             hasHeaderRow: !hasHeaderRow,
+                            skipLines,
                           }),
                         );
                       }}
@@ -1711,8 +1745,8 @@ export function ImportTransactions({ options }) {
 
 function getParseOptions(fileType, options = {}) {
   if (fileType === 'csv') {
-    const { delimiter, hasHeaderRow } = options;
-    return { delimiter, hasHeaderRow };
+    const { delimiter, hasHeaderRow, skipLines } = options;
+    return { delimiter, hasHeaderRow, skipLines };
   } else if (isOfxFile(fileType)) {
     const { fallbackMissingPayeeToMemo } = options;
     return { fallbackMissingPayeeToMemo };

packages/loot-core/src/server/accounts/parse-file.ts

@@ -18,6 +18,7 @@ type ParseFileOptions = {
   hasHeaderRow?: boolean;
   delimiter?: string;
   fallbackMissingPayeeToMemo?: boolean;
+  skipLines?: number;
 };
 
 export async function parseFile(
@@ -57,7 +58,12 @@ async function parseCSV(
   options: ParseFileOptions,
 ): Promise<ParseFileResult> {
   const errors = Array<ParseError>();
-  const contents = await fs.readFile(filepath);
+  let contents = await fs.readFile(filepath);
+
+  if (options.skipLines > 0) {
+    const lines = contents.split(/\r?\n/);
+    contents = lines.slice(options.skipLines).join('\r\n');
+  }
 
   let data;
   try {

packages/loot-core/src/types/prefs.d.ts

@@ -30,6 +30,7 @@ export type SyncedPrefs = Partial<
     [key: `parse-date-${string}-${'csv' | 'qif'}`]: string;
     [key: `csv-mappings-${string}`]: string;
     [key: `csv-delimiter-${string}`]: ',' | ';' | '\t';
+    [key: `csv-skip-lines-${string}`]: number;
     [key: `csv-has-header-${string}`]: boolean;
     [key: `ofx-fallback-missing-payee-${string}`]: boolean;
     [key: `flip-amount-${string}-${'csv' | 'qif'}`]: boolean;

upcoming-release-notes/3234.md

@@ -0,0 +1,6 @@
+---
+category: Features
+authors: [Horizon0156]
+---
+
+Added an optional configuration value to skip one or more heading lines (added by some banks, like ING) during the CSV transactions import.