fix(migration): detect header lines in csv file when importing from TickTick (#937)

pkg/modules/migration/ticktick/ticktick.go

@@ -17,6 +17,7 @@
 package ticktick
 
 import (
+	"bufio"
 	"encoding/csv"
 	"errors"
 	"io"
@@ -164,7 +165,6 @@ func (m *Migrator) Name() string {
 
 func newLineSkipDecoder(r io.Reader, linesToSkip int) gocsv.SimpleDecoder {
 	reader := csv.NewReader(r)
-	//	reader.FieldsPerRecord = -1
 	for i := 0; i < linesToSkip; i++ {
 		_, err := reader.Read()
 		if err != nil {
@@ -178,6 +178,25 @@ func newLineSkipDecoder(r io.Reader, linesToSkip int) gocsv.SimpleDecoder {
 	return gocsv.NewSimpleDecoderFromCSVReader(reader)
 }
 
+func linesToSkipBeforeHeader(file io.ReaderAt, size int64) (int, error) {
+	sr := io.NewSectionReader(file, 0, size)
+	scanner := bufio.NewScanner(sr)
+	lines := 0
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.Contains(line, "Folder Name") &&
+			strings.Contains(line, "List Name") &&
+			strings.Contains(line, "Title") {
+			break
+		}
+		lines++
+	}
+	if err := scanner.Err(); err != nil {
+		return 0, err
+	}
+	return lines, nil
+}
+
 // Migrate takes a ticktick export, parses it and imports everything in it into Vikunja.
 // @Summary Import all projects, tasks etc. from a TickTick backup export
 // @Description Imports all projects, tasks, notes, reminders, subtasks and files from a TickTick backup export into Vikunja.
@@ -220,7 +239,11 @@ func (m *Migrator) Migrate(user *user.User, file io.ReaderAt, size int64) error
 	}
 
 	allTasks := []*tickTickTask{}
-	decode := newLineSkipDecoder(fr, 3)
+	skip, err := linesToSkipBeforeHeader(file, size)
+	if err != nil {
+		return err
+	}
+	decode := newLineSkipDecoder(fr, skip)
 	err = gocsv.UnmarshalDecoder(decode, &allTasks)
 	if err != nil {
 		return err

pkg/modules/migration/ticktick/ticktick_test.go

@@ -17,10 +17,12 @@
 package ticktick
 
 import (
+	"bytes"
 	"testing"
 	"time"
 
 	"code.vikunja.io/api/pkg/models"
+	"github.com/gocarina/gocsv"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -141,3 +143,22 @@ func TestConvertTicktickTasksToVikunja(t *testing.T) {
 	assert.Equal(t, vikunjaTasks[2].Tasks[0].Title, tickTickTasks[3].Title)
 	assert.Equal(t, vikunjaTasks[2].Tasks[0].Position, tickTickTasks[3].Order)
 }
+
+func TestLinesToSkipBeforeHeader(t *testing.T) {
+	csvContent := "Date: 2024-01-01+0000\nVersion: 7.1\n" +
+		"\"Folder Name\",\"List Name\",\"Title\",\"Kind\",\"Tags\",\"Content\",\"Is Check list\",\"Start Date\",\"Due Date\",\"Reminder\",\"Repeat\",\"Priority\",\"Status\",\"Created Time\",\"Completed Time\",\"Order\",\"Timezone\",\"Is All Day\",\"Is Floating\",\"Column Name\",\"Column Order\",\"View Mode\",\"taskId\",\"parentId\"\n" +
+		",\"list\",\"task1\",\"TEXT\",\"\",\"\",\"N\",\"\",\"\",\"\",\"\",\"0\",\"0\",\"2022-10-09T15:09:48+0000\",\"\",\"-1099511627776\",\"\",\"true\",\"false\",,,\"list\",\"1\",\"\"\n"
+
+	r := bytes.NewReader([]byte(csvContent))
+	lines, err := linesToSkipBeforeHeader(r, int64(len(csvContent)))
+	require.NoError(t, err)
+	assert.Equal(t, 2, lines)
+
+	r2 := bytes.NewReader([]byte(csvContent))
+	dec := newLineSkipDecoder(r2, lines)
+	tasks := []*tickTickTask{}
+	err = gocsv.UnmarshalDecoder(dec, &tasks)
+	require.NoError(t, err)
+	require.Len(t, tasks, 1)
+	assert.Equal(t, "task1", tasks[0].Title)
+}