mirror of
https://github.com/go-vikunja/vikunja.git
synced 2025-12-05 18:57:47 -06:00
fix: TickTick import (#1871)
This change fixes a few issues with the TickTick import: 1. BOM (Byte Order Mark) Handling: Added stripBOM() function to properly handle UTF-8 BOM at the beginning of CSV files 2. Multi-line Status Section: Updated header detection to handle the multi-line status description in real TickTick exports 3. CSV Parser Configuration: Made the CSV parser more lenient with variable field counts and quote handling 4. Test Infrastructure: Added missing logger initialization for tests 5. Field Mapping: Fixed the core issue where CSV fields weren't being mapped to struct fields correctly The main problem was in the newLineSkipDecoder function where: - Header detection calculated line skip count on BOM-stripped content - CSV decoder was also stripping BOM and applying the same skip count - This caused inconsistent positioning and empty field mapping Rewrote the decoder to use a scanner-based approach with consistent BOM handling. Resolves https://github.com/go-vikunja/vikunja/issues/1870
This commit is contained in:
kolaente
GitHub
6
go.sum
6
go.sum
@@ -64,8 +64,6 @@ github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEX
|
||||
github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY=
|
||||
github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
|
||||
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
|
||||
github.com/coreos/go-oidc/v3 v3.16.0 h1:qRQUCFstKpXwmEjDQTIbyY/5jF00+asXzSkmkoa/mow=
|
||||
github.com/coreos/go-oidc/v3 v3.16.0/go.mod h1:wqPbKFrVnE90vty060SB40FCJ8fTHTxSwyXJqZH+sI8=
|
||||
github.com/coreos/go-oidc/v3 v3.17.0 h1:hWBGaQfbi0iVviX4ibC7bk8OKT5qNr4klBaCHVNvehc=
|
||||
github.com/coreos/go-oidc/v3 v3.17.0/go.mod h1:wqPbKFrVnE90vty060SB40FCJ8fTHTxSwyXJqZH+sI8=
|
||||
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
@@ -398,8 +396,6 @@ github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9Z
|
||||
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
|
||||
github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0=
|
||||
github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw=
|
||||
github.com/redis/go-redis/v9 v9.16.0 h1:OotgqgLSRCmzfqChbQyG1PHC3tLNR89DG4jdOERSEP4=
|
||||
github.com/redis/go-redis/v9 v9.16.0/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
|
||||
github.com/redis/go-redis/v9 v9.17.0 h1:K6E+ZlYN95KSMmZeEQPbU/c++wfmEvfFB17yEAq/VhM=
|
||||
github.com/redis/go-redis/v9 v9.17.0/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
@@ -521,8 +517,6 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
|
||||
golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU=
|
||||
golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc=
|
||||
golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
|
||||
golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
|
||||
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
|
||||
32
pkg/modules/migration/ticktick/main_test.go
Normal file
32
pkg/modules/migration/ticktick/main_test.go
Normal file
@@ -0,0 +1,32 @@
|
||||
// Vikunja is a to-do list application to facilitate your life.
|
||||
// Copyright 2018-present Vikunja and contributors. All rights reserved.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
package ticktick
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"code.vikunja.io/api/pkg/log"
|
||||
)
|
||||
|
||||
// TestMain is the main test function used to bootstrap the test env
|
||||
func TestMain(m *testing.M) {
|
||||
// Initialize logger for tests
|
||||
log.InitLogger()
|
||||
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
18
pkg/modules/migration/ticktick/testdata_ticktick_export.csv
Normal file
18
pkg/modules/migration/ticktick/testdata_ticktick_export.csv
Normal file
@@ -0,0 +1,18 @@
|
||||
"Date: 2025-11-25+0000"
|
||||
"Version: 7.1"
|
||||
"Status:
|
||||
0 Normal
|
||||
1 Completed
|
||||
2 Archived"
|
||||
"Folder Name","List Name","Title","Kind","Tags","Content","Is Check list","Start Date","Due Date","Reminder","Repeat","Priority","Status","Created Time","Completed Time","Order","Timezone","Is All Day","Is Floating","Column Name","Column Order","View Mode","taskId","parentId"
|
||||
"Work","Project Alpha","Task with repeating schedule","TEXT","urgent, work","This task repeats weekly","N","","","","","0","0","2022-10-09T15:09:48+0000","","-1099511627776","Europe/Berlin",,"false",,,"list","1",""
|
||||
"Work","Project Alpha","Task with reminder and dates","TEXT","work, reminder","Task description with reminder","N","2018-12-11T23:00:00+0000","2018-12-11T23:00:00+0000","","","0","0","2018-12-11T20:10:46+0000","2018-12-11T20:12:14+0000","0","Europe/Berlin","true","false",,,"list","2",""
|
||||
"Work","Project Alpha","Subtask example","TEXT","","This is a subtask","N","","","","","0","0","2022-10-09T15:20:55+0000","","1099511627776","Europe/Berlin","true","false",,,"list","3","2"
|
||||
"Work","Project Alpha","Another subtask","TEXT","","Another subtask example","N","","","","","0","0","2022-10-09T15:20:59+0000","","2199023255552","Europe/Berlin","true","false",,,"list","4","2"
|
||||
"Personal","Shopping","Buy groceries","TEXT","shopping, personal","Weekly grocery shopping","N","","","","","0","0","2018-12-29T21:48:09+0000","","-1099511627776","Europe/Berlin",,"false",,,"list","5",""
|
||||
"Personal","Shopping","Buy household items","TEXT","shopping","Cleaning supplies and toiletries","N","","","","","0","0","2018-12-29T21:48:00+0000","","-1099511627776","Europe/Berlin",,"false",,,"list","6",""
|
||||
"","Inbox","Long task description example","TEXT","","This is an example of a task with a very long description that might contain special characters and formatting.","N","","","","","0","2","2022-01-21T11:33:40+0000","2025-11-25T10:39:31+0000","-2748779069440","Europe/Berlin","false","false",,,"list","7",""
|
||||
"","Inbox","Completed task example","TEXT","","This task was completed and shows the completed timestamp","N","","","","","0","2","2022-01-21T11:33:34+0000","2025-11-25T10:39:31+0000","-2473901162496","Europe/Berlin","false","false",,,"list","8",""
|
||||
"","Inbox","Task with due date","TEXT","important, deadline","Task with specific due date and tags","N","2023-03-28T22:00:00+0000","2023-03-28T22:00:00+0000","","","0","0","2018-12-29T21:14:45+0000","","-2199023255552","Europe/Berlin","true","false",,,"list","9",""
|
||||
"","Inbox","Welcome task","TEXT","","Welcome to the task management system. This task demonstrates basic functionality.","N","2023-06-13T22:00:00+0000","2023-06-13T22:00:00+0000","","","0","0","2018-12-11T20:09:58+0000","","-1099511627776","","true","false",,,"list","10",""
|
||||
"","Inbox","Checklist example","CHECKLIST","","This is a checklist task with multiple items","Y","","","","","0","0","2018-12-11T20:09:58+0000","","2199023255552","",,"false",,,"list","11",""
|
||||
|
Can't render this file because it has a wrong number of fields in line 7.
|
@@ -0,0 +1,14 @@
|
||||
"Date: 2025-11-25+0000"
|
||||
"Version: 7.1"
|
||||
"Status:
|
||||
0 Normal
|
||||
1 Completed
|
||||
2 Archived"
|
||||
"Folder Name","List Name","Title","Kind","Tags","Content","Is Check list","Start Date","Due Date","Reminder","Repeat","Priority","Status","Created Time","Completed Time","Order","Timezone","Is All Day","Is Floating","Column Name","Column Order","View Mode","taskId","parentId"
|
||||
"Work","Project Alpha","Task with multiline
|
||||
description","TEXT","urgent, work","This is a task description
|
||||
that spans multiple lines.
|
||||
|
||||
It has paragraphs and everything!
|
||||
Including special characters: #, *, @","N","","","","","0","0","2022-10-09T15:09:48+0000","","-1099511627776","Europe/Berlin",,"false",,,"list","1",""
|
||||
"Work","Project Alpha","Regular task","TEXT","","Simple description","N","","","","","0","0","2022-10-09T15:10:00+0000","","-1099511627775","Europe/Berlin",,"false",,,"list","2",""
|
||||
|
Can't render this file because it has a wrong number of fields in line 7.
|
@@ -18,6 +18,7 @@ package ticktick
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/csv"
|
||||
"errors"
|
||||
"io"
|
||||
@@ -25,7 +26,6 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"code.vikunja.io/api/pkg/log"
|
||||
"code.vikunja.io/api/pkg/models"
|
||||
"code.vikunja.io/api/pkg/modules/migration"
|
||||
"code.vikunja.io/api/pkg/user"
|
||||
@@ -101,9 +101,13 @@ func convertTickTickToVikunja(tasks []*tickTickTask) (result []*models.ProjectWi
|
||||
|
||||
labels := make([]*models.Label, 0, len(t.Tags))
|
||||
for _, tag := range t.Tags {
|
||||
labels = append(labels, &models.Label{
|
||||
Title: tag,
|
||||
})
|
||||
// Only create labels for non-empty tags after trimming whitespace
|
||||
trimmedTag := strings.TrimSpace(tag)
|
||||
if trimmedTag != "" {
|
||||
labels = append(labels, &models.Label{
|
||||
Title: trimmedTag,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
task := &models.TaskWithComments{
|
||||
@@ -163,24 +167,79 @@ func (m *Migrator) Name() string {
|
||||
return "ticktick"
|
||||
}
|
||||
|
||||
func newLineSkipDecoder(r io.Reader, linesToSkip int) gocsv.SimpleDecoder {
|
||||
reader := csv.NewReader(r)
|
||||
for i := 0; i < linesToSkip; i++ {
|
||||
_, err := reader.Read()
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
// stripBOM removes the UTF-8 BOM from the beginning of a reader
|
||||
func stripBOM(r io.Reader) io.Reader {
|
||||
// Read the first few bytes to check for BOM
|
||||
buf := make([]byte, 3)
|
||||
n, err := r.Read(buf)
|
||||
if err != nil && err != io.EOF {
|
||||
// If we read some bytes before the error, preserve them
|
||||
if n > 0 {
|
||||
return io.MultiReader(bytes.NewReader(buf[:n]), r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// Check if it starts with UTF-8 BOM (0xEF, 0xBB, 0xBF)
|
||||
// We need exactly 3 bytes and they must match the BOM sequence
|
||||
if n == 3 && len(buf) >= 3 && buf[0] == 0xEF && buf[1] == 0xBB && buf[2] == 0xBF {
|
||||
// BOM found, return reader without BOM
|
||||
return io.MultiReader(bytes.NewReader(buf[3:n]), r)
|
||||
}
|
||||
|
||||
// No BOM found, return reader with the bytes we read back
|
||||
return io.MultiReader(bytes.NewReader(buf[:n]), r)
|
||||
}
|
||||
|
||||
func newLineSkipDecoder(r io.Reader, linesToSkip int) (gocsv.SimpleDecoder, error) {
|
||||
// Strip BOM if present - this must be done consistently with linesToSkipBeforeHeader
|
||||
r = stripBOM(r)
|
||||
|
||||
// Read all content into memory so we can work with it
|
||||
// This is acceptable since CSV imports are typically not huge files
|
||||
allBytes, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Skip the metadata lines before the CSV header by finding newlines
|
||||
// linesToSkipBeforeHeader counts raw text lines (newlines), not CSV records,
|
||||
// because even metadata can have multiline quoted fields.
|
||||
// We manually search for newlines (no buffer size limits like bufio.Scanner)
|
||||
bytesSkipped := 0
|
||||
linesFound := 0
|
||||
for i := 0; i < len(allBytes) && linesFound < linesToSkip; i++ {
|
||||
if allBytes[i] == '\n' {
|
||||
linesFound++
|
||||
if linesFound == linesToSkip {
|
||||
// Position is right after the Nth newline
|
||||
bytesSkipped = i + 1
|
||||
break
|
||||
}
|
||||
log.Debugf("[TickTick Migration] CSV parse error: %s", err)
|
||||
}
|
||||
}
|
||||
reader.FieldsPerRecord = 0
|
||||
return gocsv.NewSimpleDecoderFromCSVReader(reader)
|
||||
|
||||
if linesFound < linesToSkip {
|
||||
return nil, io.ErrUnexpectedEOF
|
||||
}
|
||||
|
||||
// Now create a CSV reader starting from after the skipped lines
|
||||
// The CSV reader will properly handle any multiline quoted fields in the actual data
|
||||
remainingContent := allBytes[bytesSkipped:]
|
||||
reader := csv.NewReader(bytes.NewReader(remainingContent))
|
||||
|
||||
// Allow variable field counts and be lenient with parsing
|
||||
reader.FieldsPerRecord = -1
|
||||
reader.LazyQuotes = true
|
||||
reader.TrimLeadingSpace = true
|
||||
return gocsv.NewSimpleDecoderFromCSVReader(reader), nil
|
||||
}
|
||||
|
||||
func linesToSkipBeforeHeader(file io.ReaderAt, size int64) (int, error) {
|
||||
sr := io.NewSectionReader(file, 0, size)
|
||||
scanner := bufio.NewScanner(sr)
|
||||
// Strip BOM before scanning for header
|
||||
r := stripBOM(sr)
|
||||
scanner := bufio.NewScanner(r)
|
||||
lines := 0
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
@@ -243,7 +302,10 @@ func (m *Migrator) Migrate(user *user.User, file io.ReaderAt, size int64) error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
decode := newLineSkipDecoder(fr, skip)
|
||||
decode, err := newLineSkipDecoder(fr, skip)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = gocsv.UnmarshalDecoder(decode, &allTasks)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -17,7 +17,12 @@
|
||||
package ticktick
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/csv"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -155,10 +160,417 @@ func TestLinesToSkipBeforeHeader(t *testing.T) {
|
||||
assert.Equal(t, 2, lines)
|
||||
|
||||
r2 := bytes.NewReader([]byte(csvContent))
|
||||
dec := newLineSkipDecoder(r2, lines)
|
||||
dec, err := newLineSkipDecoder(r2, lines)
|
||||
require.NoError(t, err)
|
||||
tasks := []*tickTickTask{}
|
||||
err = gocsv.UnmarshalDecoder(dec, &tasks)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, tasks, 1)
|
||||
assert.Equal(t, "task1", tasks[0].Title)
|
||||
}
|
||||
|
||||
func TestLinesToSkipBeforeHeaderWithRealCSV(t *testing.T) {
|
||||
// This is the actual format from a real TickTick export with BOM and multi-line status
|
||||
csvContent := "\uFEFF\"Date: 2025-11-25+0000\"\n" +
|
||||
"\"Version: 7.1\"\n" +
|
||||
"\"Status: \n" +
|
||||
"0 Normal\n" +
|
||||
"1 Completed\n" +
|
||||
"2 Archived\"\n" +
|
||||
"\"Folder Name\",\"List Name\",\"Title\",\"Kind\",\"Tags\",\"Content\",\"Is Check list\",\"Start Date\",\"Due Date\",\"Reminder\",\"Repeat\",\"Priority\",\"Status\",\"Created Time\",\"Completed Time\",\"Order\",\"Timezone\",\"Is All Day\",\"Is Floating\",\"Column Name\",\"Column Order\",\"View Mode\",\"taskId\",\"parentId\"\n" +
|
||||
"\"dsx\",\"x\",\"this task repeats\",\"TEXT\",\"\",\"\",\"N\",\"\",\"\",\"\",\"\",\"0\",\"0\",\"2022-10-09T15:09:48+0000\",\"\",\"-1099511627776\",\"Europe/Berlin\",,\"false\",,,\"list\",\"2\",\"\"\n"
|
||||
|
||||
t.Logf("CSV content length: %d", len(csvContent))
|
||||
t.Logf("CSV content first 100 chars: %q", csvContent[:100])
|
||||
|
||||
r := bytes.NewReader([]byte(csvContent))
|
||||
lines, err := linesToSkipBeforeHeader(r, int64(len(csvContent)))
|
||||
require.NoError(t, err)
|
||||
t.Logf("Lines to skip: %d", lines)
|
||||
assert.Equal(t, 6, lines) // Should skip 6 lines to get to the header
|
||||
|
||||
r2 := bytes.NewReader([]byte(csvContent))
|
||||
dec, err := newLineSkipDecoder(r2, lines)
|
||||
require.NoError(t, err)
|
||||
tasks := []*tickTickTask{}
|
||||
err = gocsv.UnmarshalDecoder(dec, &tasks)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, tasks, 1)
|
||||
assert.Equal(t, "this task repeats", tasks[0].Title)
|
||||
assert.Equal(t, "dsx", tasks[0].FolderName)
|
||||
assert.Equal(t, "x", tasks[0].ProjectName)
|
||||
}
|
||||
|
||||
func TestLinesToSkipBeforeHeaderWithCleanTestFile(t *testing.T) {
|
||||
// Test with the cleaned-up test CSV file
|
||||
file, err := os.Open("testdata_ticktick_export.csv")
|
||||
require.NoError(t, err)
|
||||
defer file.Close()
|
||||
|
||||
stat, err := file.Stat()
|
||||
require.NoError(t, err)
|
||||
|
||||
lines, err := linesToSkipBeforeHeader(file, stat.Size())
|
||||
require.NoError(t, err)
|
||||
t.Logf("Lines to skip in test file: %d", lines)
|
||||
assert.Equal(t, 6, lines) // Should skip 6 lines to get to the header
|
||||
|
||||
// Reset file position
|
||||
_, err = file.Seek(0, io.SeekStart)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Let's manually check what the header line looks like after skipping
|
||||
r := stripBOM(file)
|
||||
scanner := bufio.NewScanner(r)
|
||||
for i := 0; i <= lines; i++ {
|
||||
if !scanner.Scan() {
|
||||
break
|
||||
}
|
||||
if i == lines {
|
||||
t.Logf("Header line after skipping %d lines: %q", lines, scanner.Text())
|
||||
}
|
||||
}
|
||||
|
||||
// Reset file position again
|
||||
_, err = file.Seek(0, io.SeekStart)
|
||||
require.NoError(t, err)
|
||||
|
||||
dec, err := newLineSkipDecoder(file, lines)
|
||||
require.NoError(t, err)
|
||||
tasks := []*tickTickTask{}
|
||||
err = gocsv.UnmarshalDecoder(dec, &tasks)
|
||||
require.NoError(t, err)
|
||||
require.Greater(t, len(tasks), 0)
|
||||
|
||||
// Verify that the first task has actual data
|
||||
assert.Equal(t, "Work", tasks[0].FolderName)
|
||||
assert.Equal(t, "Project Alpha", tasks[0].ProjectName)
|
||||
assert.Equal(t, "Task with repeating schedule", tasks[0].Title)
|
||||
}
|
||||
|
||||
func TestBOMStripping(t *testing.T) {
|
||||
// Test BOM stripping specifically
|
||||
csvWithBOM := "\uFEFF\"Folder Name\",\"List Name\",\"Title\"\n\"test\",\"list\",\"task\"\n"
|
||||
|
||||
r := stripBOM(bytes.NewReader([]byte(csvWithBOM)))
|
||||
scanner := bufio.NewScanner(r)
|
||||
|
||||
// Read first line (header)
|
||||
require.True(t, scanner.Scan())
|
||||
header := scanner.Text()
|
||||
t.Logf("Header after BOM stripping: %q", header)
|
||||
|
||||
// Read second line (data)
|
||||
require.True(t, scanner.Scan())
|
||||
data := scanner.Text()
|
||||
t.Logf("Data line: %q", data)
|
||||
|
||||
// Test CSV parsing
|
||||
r2 := stripBOM(bytes.NewReader([]byte(csvWithBOM)))
|
||||
reader := csv.NewReader(r2)
|
||||
records, err := reader.ReadAll()
|
||||
require.NoError(t, err)
|
||||
require.Len(t, records, 2)
|
||||
t.Logf("CSV records: %+v", records)
|
||||
}
|
||||
|
||||
func TestEmptyLabelHandling(t *testing.T) {
|
||||
t.Run("Normal tags", func(t *testing.T) {
|
||||
task := &tickTickTask{
|
||||
Title: "Test Task",
|
||||
ProjectName: "Test Project",
|
||||
TagsList: "work, personal, urgent",
|
||||
}
|
||||
task.Tags = strings.Split(task.TagsList, ", ")
|
||||
|
||||
vikunjaTasks := convertTickTickToVikunja([]*tickTickTask{task})
|
||||
projectWithTasks := findProjectWithTasks(t, vikunjaTasks)
|
||||
vikunjaTask := projectWithTasks.Tasks[0]
|
||||
|
||||
expectedTags := []string{"work", "personal", "urgent"}
|
||||
assertLabelsMatch(t, vikunjaTask, expectedTags)
|
||||
})
|
||||
|
||||
t.Run("Tags with extra spaces", func(t *testing.T) {
|
||||
task := &tickTickTask{
|
||||
Title: "Test Task",
|
||||
ProjectName: "Test Project",
|
||||
TagsList: "work, personal , urgent",
|
||||
}
|
||||
task.Tags = strings.Split(task.TagsList, ", ")
|
||||
|
||||
vikunjaTasks := convertTickTickToVikunja([]*tickTickTask{task})
|
||||
projectWithTasks := findProjectWithTasks(t, vikunjaTasks)
|
||||
vikunjaTask := projectWithTasks.Tasks[0]
|
||||
|
||||
expectedTags := []string{"work", "personal", "urgent"}
|
||||
assertLabelsMatch(t, vikunjaTask, expectedTags)
|
||||
})
|
||||
|
||||
t.Run("Empty tags mixed with valid ones", func(t *testing.T) {
|
||||
task := &tickTickTask{
|
||||
Title: "Test Task",
|
||||
ProjectName: "Test Project",
|
||||
TagsList: "work, , urgent, ",
|
||||
}
|
||||
task.Tags = strings.Split(task.TagsList, ", ")
|
||||
|
||||
vikunjaTasks := convertTickTickToVikunja([]*tickTickTask{task})
|
||||
projectWithTasks := findProjectWithTasks(t, vikunjaTasks)
|
||||
vikunjaTask := projectWithTasks.Tasks[0]
|
||||
|
||||
expectedTags := []string{"work", "urgent"}
|
||||
assertLabelsMatch(t, vikunjaTask, expectedTags)
|
||||
})
|
||||
|
||||
t.Run("Only whitespace tags", func(t *testing.T) {
|
||||
task := &tickTickTask{
|
||||
Title: "Test Task",
|
||||
ProjectName: "Test Project",
|
||||
TagsList: " , , ",
|
||||
}
|
||||
task.Tags = strings.Split(task.TagsList, ", ")
|
||||
|
||||
vikunjaTasks := convertTickTickToVikunja([]*tickTickTask{task})
|
||||
projectWithTasks := findProjectWithTasks(t, vikunjaTasks)
|
||||
vikunjaTask := projectWithTasks.Tasks[0]
|
||||
|
||||
expectedTags := []string{}
|
||||
assertLabelsMatch(t, vikunjaTask, expectedTags)
|
||||
})
|
||||
|
||||
t.Run("Empty string", func(t *testing.T) {
|
||||
task := &tickTickTask{
|
||||
Title: "Test Task",
|
||||
ProjectName: "Test Project",
|
||||
TagsList: "",
|
||||
}
|
||||
task.Tags = strings.Split(task.TagsList, ", ")
|
||||
|
||||
vikunjaTasks := convertTickTickToVikunja([]*tickTickTask{task})
|
||||
projectWithTasks := findProjectWithTasks(t, vikunjaTasks)
|
||||
vikunjaTask := projectWithTasks.Tasks[0]
|
||||
|
||||
expectedTags := []string{}
|
||||
assertLabelsMatch(t, vikunjaTask, expectedTags)
|
||||
})
|
||||
|
||||
t.Run("Single valid tag", func(t *testing.T) {
|
||||
task := &tickTickTask{
|
||||
Title: "Test Task",
|
||||
ProjectName: "Test Project",
|
||||
TagsList: "important",
|
||||
}
|
||||
task.Tags = strings.Split(task.TagsList, ", ")
|
||||
|
||||
vikunjaTasks := convertTickTickToVikunja([]*tickTickTask{task})
|
||||
projectWithTasks := findProjectWithTasks(t, vikunjaTasks)
|
||||
vikunjaTask := projectWithTasks.Tasks[0]
|
||||
|
||||
expectedTags := []string{"important"}
|
||||
assertLabelsMatch(t, vikunjaTask, expectedTags)
|
||||
})
|
||||
|
||||
t.Run("Single empty tag", func(t *testing.T) {
|
||||
task := &tickTickTask{
|
||||
Title: "Test Task",
|
||||
ProjectName: "Test Project",
|
||||
TagsList: " ",
|
||||
}
|
||||
task.Tags = strings.Split(task.TagsList, ", ")
|
||||
|
||||
vikunjaTasks := convertTickTickToVikunja([]*tickTickTask{task})
|
||||
projectWithTasks := findProjectWithTasks(t, vikunjaTasks)
|
||||
vikunjaTask := projectWithTasks.Tasks[0]
|
||||
|
||||
expectedTags := []string{}
|
||||
assertLabelsMatch(t, vikunjaTask, expectedTags)
|
||||
})
|
||||
|
||||
t.Run("Tags with leading/trailing spaces", func(t *testing.T) {
|
||||
task := &tickTickTask{
|
||||
Title: "Test Task",
|
||||
ProjectName: "Test Project",
|
||||
TagsList: " work , personal, urgent ",
|
||||
}
|
||||
task.Tags = strings.Split(task.TagsList, ", ")
|
||||
|
||||
vikunjaTasks := convertTickTickToVikunja([]*tickTickTask{task})
|
||||
projectWithTasks := findProjectWithTasks(t, vikunjaTasks)
|
||||
vikunjaTask := projectWithTasks.Tasks[0]
|
||||
|
||||
expectedTags := []string{"work", "personal", "urgent"}
|
||||
assertLabelsMatch(t, vikunjaTask, expectedTags)
|
||||
})
|
||||
}
|
||||
|
||||
// Helper function to find the project that contains tasks
|
||||
func findProjectWithTasks(t *testing.T, vikunjaTasks []*models.ProjectWithTasksAndBuckets) *models.ProjectWithTasksAndBuckets {
|
||||
t.Helper()
|
||||
|
||||
// The function creates a parent project and child projects
|
||||
// We expect 2 projects: parent "Migrated from TickTick" and child "Test Project"
|
||||
require.Len(t, vikunjaTasks, 2)
|
||||
|
||||
// Find the project with tasks (should be the child project)
|
||||
for _, project := range vikunjaTasks {
|
||||
if len(project.Tasks) > 0 {
|
||||
require.Len(t, project.Tasks, 1)
|
||||
return project
|
||||
}
|
||||
}
|
||||
|
||||
t.Fatal("Should find a project with tasks")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Helper function to assert that labels match expected tags
|
||||
func assertLabelsMatch(t *testing.T, vikunjaTask *models.TaskWithComments, expectedTags []string) {
|
||||
t.Helper()
|
||||
|
||||
// Check that only non-empty labels were created
|
||||
assert.Len(t, vikunjaTask.Labels, len(expectedTags), "Number of labels should match expected")
|
||||
|
||||
// Check that the label titles match expected tags
|
||||
actualTags := make([]string, len(vikunjaTask.Labels))
|
||||
for i, label := range vikunjaTask.Labels {
|
||||
actualTags[i] = label.Title
|
||||
}
|
||||
|
||||
assert.ElementsMatch(t, expectedTags, actualTags, "Label titles should match expected tags")
|
||||
|
||||
// Ensure no empty labels were created
|
||||
for _, label := range vikunjaTask.Labels {
|
||||
assert.NotEmpty(t, strings.TrimSpace(label.Title), "No label should be empty or whitespace-only")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultilineDescriptions(t *testing.T) {
|
||||
// Test with a CSV fixture that contains actual multiline content in quoted fields
|
||||
file, err := os.Open("testdata_ticktick_multiline.csv")
|
||||
require.NoError(t, err, "Failed to open test fixture")
|
||||
defer file.Close()
|
||||
|
||||
stat, err := file.Stat()
|
||||
require.NoError(t, err)
|
||||
|
||||
lines, err := linesToSkipBeforeHeader(file, stat.Size())
|
||||
require.NoError(t, err)
|
||||
t.Logf("Lines to skip: %d", lines)
|
||||
assert.Equal(t, 6, lines, "Should skip 6 metadata lines")
|
||||
|
||||
// Reset file position
|
||||
_, err = file.Seek(0, io.SeekStart)
|
||||
require.NoError(t, err)
|
||||
|
||||
dec, err := newLineSkipDecoder(file, lines)
|
||||
require.NoError(t, err)
|
||||
tasks := []*tickTickTask{}
|
||||
err = gocsv.UnmarshalDecoder(dec, &tasks)
|
||||
require.NoError(t, err, "Failed to parse CSV with multiline descriptions")
|
||||
|
||||
// We expect 2 tasks in this fixture
|
||||
require.Len(t, tasks, 2, "Should parse exactly 2 tasks")
|
||||
|
||||
// First task has multiline content in both Title and Content fields
|
||||
task1 := tasks[0]
|
||||
assert.Equal(t, "Work", task1.FolderName)
|
||||
assert.Equal(t, "Project Alpha", task1.ProjectName)
|
||||
|
||||
// The title contains a newline
|
||||
assert.Contains(t, task1.Title, "Task with multiline")
|
||||
assert.Contains(t, task1.Title, "description")
|
||||
assert.Contains(t, task1.Title, "\n", "Title should contain actual newline character")
|
||||
|
||||
// The content contains multiple newlines and paragraphs
|
||||
assert.Contains(t, task1.Content, "This is a task description")
|
||||
assert.Contains(t, task1.Content, "that spans multiple lines")
|
||||
assert.Contains(t, task1.Content, "It has paragraphs and everything!")
|
||||
assert.Contains(t, task1.Content, "Including special characters: #, *, @")
|
||||
|
||||
// Count newlines in content - should have at least 3 (between the 4 lines)
|
||||
newlineCount := strings.Count(task1.Content, "\n")
|
||||
assert.GreaterOrEqual(t, newlineCount, 3, "Content should have multiple newlines")
|
||||
|
||||
// Second task is a regular task without multiline content
|
||||
task2 := tasks[1]
|
||||
assert.Equal(t, "Regular task", task2.Title)
|
||||
assert.Equal(t, "Simple description", task2.Content)
|
||||
assert.NotContains(t, task2.Title, "\n", "Regular task title should not have newlines")
|
||||
|
||||
t.Logf("Successfully parsed tasks with multiline content:")
|
||||
t.Logf(" Task 1 title: %q", task1.Title)
|
||||
t.Logf(" Task 1 content: %q", task1.Content)
|
||||
t.Logf(" Task 2 title: %q", task2.Title)
|
||||
}
|
||||
|
||||
func TestEmptyLabelHandlingWithRealCSV(t *testing.T) {
|
||||
t.Run("Parse CSV file", func(t *testing.T) {
|
||||
file, err := os.Open("testdata_ticktick_export.csv")
|
||||
require.NoError(t, err)
|
||||
defer file.Close()
|
||||
|
||||
stat, err := file.Stat()
|
||||
require.NoError(t, err)
|
||||
|
||||
lines, err := linesToSkipBeforeHeader(file, stat.Size())
|
||||
require.NoError(t, err)
|
||||
|
||||
// Reset file position
|
||||
_, err = file.Seek(0, io.SeekStart)
|
||||
require.NoError(t, err)
|
||||
|
||||
dec, err := newLineSkipDecoder(file, lines)
|
||||
require.NoError(t, err)
|
||||
tasks := []*tickTickTask{}
|
||||
err = gocsv.UnmarshalDecoder(dec, &tasks)
|
||||
require.NoError(t, err)
|
||||
require.Greater(t, len(tasks), 0)
|
||||
|
||||
t.Logf("Successfully parsed %d tasks from CSV file", len(tasks))
|
||||
})
|
||||
|
||||
t.Run("Process tags and check for empty labels", func(t *testing.T) {
|
||||
file, err := os.Open("testdata_ticktick_export.csv")
|
||||
require.NoError(t, err)
|
||||
defer file.Close()
|
||||
|
||||
stat, err := file.Stat()
|
||||
require.NoError(t, err)
|
||||
|
||||
lines, err := linesToSkipBeforeHeader(file, stat.Size())
|
||||
require.NoError(t, err)
|
||||
|
||||
// Reset file position
|
||||
_, err = file.Seek(0, io.SeekStart)
|
||||
require.NoError(t, err)
|
||||
|
||||
dec, err := newLineSkipDecoder(file, lines)
|
||||
require.NoError(t, err)
|
||||
tasks := []*tickTickTask{}
|
||||
err = gocsv.UnmarshalDecoder(dec, &tasks)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Process tags as the migration code does
|
||||
for _, task := range tasks {
|
||||
task.Tags = strings.Split(task.TagsList, ", ")
|
||||
}
|
||||
|
||||
// Convert to Vikunja format
|
||||
vikunjaTasks := convertTickTickToVikunja(tasks)
|
||||
|
||||
// Check all tasks for empty labels
|
||||
totalLabels := 0
|
||||
for _, project := range vikunjaTasks {
|
||||
for _, task := range project.Tasks {
|
||||
totalLabels += len(task.Labels)
|
||||
for _, label := range task.Labels {
|
||||
assert.NotEmpty(t, strings.TrimSpace(label.Title),
|
||||
"No label should be empty or whitespace-only. Found empty label in task: %s", task.Title)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
t.Logf("Successfully processed %d tasks with %d total labels, no empty labels created", len(tasks), totalLabels)
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user