Files
vikunja/pkg/modules/migration/create_from_structure.go
kolaenteandTink 7f9f3e091f fix(migration): log migration start and completion at info level
A background import that never finished left no trace in a default log
setup, since start and completion were only logged at debug.
2026-09-15 16:59:37 +02:00

804 lines
23 KiB
Go

// Vikunja is a to-do list application to facilitate your life.
// Copyright 2018-present Vikunja and contributors. All rights reserved.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package migration
import (
"bytes"
"context"
"errors"
"io"
"io/fs"
"math"
"strings"
"time"
"xorm.io/xorm"
"code.vikunja.io/api/pkg/db"
"code.vikunja.io/api/pkg/events"
"code.vikunja.io/api/pkg/files"
"code.vikunja.io/api/pkg/log"
"code.vikunja.io/api/pkg/models"
"code.vikunja.io/api/pkg/modules/background/handler"
"code.vikunja.io/api/pkg/user"
"code.vikunja.io/api/pkg/utils"
)
// FileProvider opens attachment and background bytes lazily.
// Readers must be seekable; the second return value is their size in bytes.
type FileProvider interface {
OpenAttachment(attachment *models.TaskAttachment) (io.ReadSeekCloser, int64, error)
OpenBackground(project *models.ProjectWithTasksAndBuckets) (io.ReadSeekCloser, int64, error)
}
type backgroundFileStorageCounter interface {
CountBackgroundFile(size int64) error
}
// InsertFromStructure takes a fully nested Vikunja data structure and a user and then creates everything for this user
// (Projects, tasks, etc. Even attachments and relations.)
func InsertFromStructure(str []*models.ProjectWithTasksAndBuckets, u *user.User) (err error) {
return insertFromStructureWithFileProvider(str, u, nil)
}
// InsertFromStructureWithFileProvider imports lazily opened attachment and background bytes.
func InsertFromStructureWithFileProvider(str []*models.ProjectWithTasksAndBuckets, u *user.User, provider FileProvider) (err error) {
return insertFromStructureWithFileProvider(str, u, provider)
}
func insertFromStructureWithFileProvider(str []*models.ProjectWithTasksAndBuckets, u *user.User, provider FileProvider) (err error) {
s := db.NewSession()
defer s.Close()
// Callers may pass a user built from jwt claims; load the stored one so
// assignee matching sees the current email/username.
importer, err := user.GetUserWithEmail(s, &user.User{ID: u.ID})
if err != nil {
return err
}
// Failed transactions roll back file rows, not blobs written before commit.
createdFiles := &[]int64{}
err = insertFromStructure(s, str, importer, provider, createdFiles)
if err != nil {
log.Errorf("[creating structure] Error while creating structure: %s", err.Error())
cleanupAndRollback(s, *createdFiles)
events.CleanupPending(s)
return err
}
err = s.Commit()
if err != nil {
events.CleanupPending(s)
return err
}
events.DispatchPending(context.Background(), s)
return nil
}
func cleanupAndRollback(s *xorm.Session, fileIDs []int64) {
cleanupCreatedFiles(fileIDs)
_ = s.Rollback()
}
// Delete blobs before rollback releases their reusable database IDs.
func cleanupCreatedFiles(fileIDs []int64) {
for _, id := range fileIDs {
if err := files.DeleteBlob(id); err != nil {
if errors.Is(err, fs.ErrNotExist) {
continue
}
log.Errorf("[creating structure] Could not clean up file %d of a failed import: %s", id, err)
}
}
}
func insertFromStructure(s *xorm.Session, str []*models.ProjectWithTasksAndBuckets, user *user.User, provider FileProvider, createdFiles *[]int64) (err error) {
log.Infof("[creating structure] Creating %d projects for user %d", len(str), user.ID)
// Seed the dedup map with the user's existing labels so re-imports
// reuse them instead of creating duplicates (see issue #2742).
labels := make(map[string]*models.Label)
existingLabels := []*models.Label{}
if err = s.Where("created_by_id = ?", user.ID).Find(&existingLabels); err != nil {
return err
}
for _, l := range existingLabels {
labels[l.Title+utils.NormalizeHex(l.HexColor)] = l
}
archivedProjects := []int64{}
childRelations := make(map[int64][]int64) // old id is the key, slice of old children ids
projectsByOldID := make(map[int64]*models.Project) // old id is the key
// Create all projects
for i, p := range str {
if p.ID == models.FavoritesPseudoProjectID {
continue
}
oldID := p.ID
if p.ParentProjectID != nil && *p.ParentProjectID != 0 {
childRelations[*p.ParentProjectID] = append(childRelations[*p.ParentProjectID], oldID)
p.ParentProjectID = nil
}
p.ID = 0
for _, view := range p.Views {
view.ProjectID = 0
}
err = createProject(s, p, &archivedProjects, labels, user, provider, createdFiles)
if err != nil {
return err
}
projectsByOldID[oldID] = &str[i].Project
}
// parent / child relations
for parentID, children := range childRelations {
parent, has := projectsByOldID[parentID]
if !has {
log.Debugf("[creating structure] could not find parentID project with old id %d", parentID)
continue
}
for _, childID := range children {
child, has := projectsByOldID[childID]
if !has {
log.Debugf("[creating structure] could not find child project with old id %d for parent project with old id %d", childID, parentID)
continue
}
child.ParentProjectID = models.Ptr(parent.ID)
err = child.Update(s, user)
if err != nil {
return err
}
}
}
if len(archivedProjects) > 0 {
_, err = s.
Cols("is_archived").
In("id", archivedProjects).
Update(&models.Project{IsArchived: true})
if err != nil {
return err
}
}
// Exports written before archiving cascaded carry unflagged children under archived parents.
for _, projectID := range archivedProjects {
err = models.SetArchiveStateForProjectDescendants(s, projectID, true)
if err != nil {
return err
}
}
log.Infof("[creating structure] Done inserting new task structure")
return nil
}
// seedMissingTaskPositions numbers the tasks of an export that carries no order
// information at all. A task without a position is inserted in front of the lowest one
// by halving it, which hits the minimum spacing every few dozen inserts and then
// recalculates every position in the view - O(n²) for a large import (#3297).
// Seeding also keeps the order the export was written in.
func seedMissingTaskPositions(tasks []*models.TaskWithComments) {
for _, t := range tasks {
if t.Position != 0 {
return
}
}
for i, t := range tasks {
t.Position = float64(i+1) * math.Pow(2, 16)
}
}
func createProject(s *xorm.Session, project *models.ProjectWithTasksAndBuckets, archivedProjectIDs *[]int64, labels map[string]*models.Label, user *user.User, provider FileProvider, createdFiles *[]int64) (err error) {
err = createProjectWithEverything(s, project, archivedProjectIDs, labels, user, provider, createdFiles)
if err != nil {
return err
}
log.Debugf("[creating structure] Created project %d", project.ID)
return
}
func createProjectWithEverything(s *xorm.Session, project *models.ProjectWithTasksAndBuckets, archivedProjects *[]int64, labels map[string]*models.Label, user *user.User, provider FileProvider, createdFiles *[]int64) (err error) {
// The tasks and bucket slices are going to be reset during the creation of the project, so we rescue it here
// to be able to still loop over them aftere the project was created.
tasks := project.Tasks
originalBuckets := project.Buckets
originalBackgroundInformation := project.BackgroundInformation
needsDefaultBucket := false
oldViews := project.Views
// Saving the archived status to archive the project again after creating it
var wasArchived bool
if project.IsArchived {
wasArchived = true
project.IsArchived = false
}
project.ID = 0
project.Project.BackgroundFileID = 0
err = models.CreateProject(s, &project.Project, user, false, false)
if err != nil && models.IsErrProjectIdentifierIsNotUnique(err) {
project.Identifier = ""
err = models.CreateProject(s, &project.Project, user, false, false)
}
if err != nil {
return
}
if wasArchived {
*archivedProjects = append(*archivedProjects, project.ID)
}
log.Debugf("[creating structure] Created project %d", project.ID)
bf, is := originalBackgroundInformation.(*bytes.Buffer)
if is {
backgroundFile := bytes.NewReader(bf.Bytes())
log.Debugf("[creating structure] Creating a background file for project %d", project.ID)
_, err = handler.SaveBackgroundFile(s, user, &project.Project, backgroundFile, "")
if err != nil {
log.Errorf("[creating structure] Could not create background for project %d, error was %v", project.ID, err)
} else {
*createdFiles = append(*createdFiles, project.Project.BackgroundFileID)
}
log.Debugf("[creating structure] Created a background file for project %d", project.ID)
} else if provider != nil {
backgroundFile, _, perr := provider.OpenBackground(project)
if perr != nil {
return perr
}
if backgroundFile != nil {
log.Debugf("[creating structure] Creating a background file for project %d", project.ID)
var storedSize int64
storedSize, err = handler.SaveBackgroundFile(s, user, &project.Project, backgroundFile, "")
closeErr := backgroundFile.Close()
if err != nil {
return err
}
*createdFiles = append(*createdFiles, project.Project.BackgroundFileID)
if counter, ok := provider.(backgroundFileStorageCounter); ok {
if err = counter.CountBackgroundFile(storedSize); err != nil {
return err
}
}
if closeErr != nil {
return closeErr
}
log.Debugf("[creating structure] Created a background file for project %d", project.ID)
}
}
// Create all buckets
bucketsByOldID := make(map[int64]*models.Bucket) // old bucket id is the key
if len(project.Buckets) > 0 {
log.Debugf("[creating structure] Creating %d buckets", len(project.Buckets))
}
for _, bucket := range originalBuckets {
if _, exists := bucketsByOldID[bucket.ID]; exists {
continue
}
oldID := bucket.ID
bucket.ID = 0 // We want a new id
bucket.ProjectID = project.ID
err = bucket.Create(s, user)
if err != nil {
return
}
bucketsByOldID[oldID] = bucket
log.Debugf("[creating structure] Created bucket %d, old ID was %d", bucket.ID, oldID)
}
// project_view_id is intentionally not writable through Bucket.Update
// (blocks cross-tenant relocation, GHSA-569v). The importer legitimately
// remaps buckets onto same-project views, so persist that column directly.
persistBucketView := func(b *models.Bucket) error {
_, err := s.Where("id = ?", b.ID).Cols("project_view_id").Update(b)
return err
}
// Create all views, create default views if we don't have any
viewsByOldIDs := make(map[int64]*models.ProjectView, len(oldViews))
if len(oldViews) > 0 {
for _, view := range oldViews {
oldID := view.ID
view.ID = 0
if view.DefaultBucketID != 0 {
bucket, has := bucketsByOldID[view.DefaultBucketID]
if has {
view.DefaultBucketID = bucket.ID
}
}
if view.DoneBucketID != 0 {
bucket, has := bucketsByOldID[view.DoneBucketID]
if has {
view.DoneBucketID = bucket.ID
}
}
view.ProjectID = project.ID
err = view.Create(s, user)
if err != nil {
return
}
viewsByOldIDs[oldID] = view
}
for oldID, bucket := range bucketsByOldID {
newView, has := viewsByOldIDs[bucket.ProjectViewID]
if !has {
err = bucket.Delete(s, user)
if err != nil {
return
}
delete(bucketsByOldID, oldID)
continue
}
bucket.ProjectViewID = newView.ID
err = persistBucketView(bucket)
if err != nil {
return
}
}
} else {
if len(project.Views) == 0 {
err = models.CreateDefaultViewsForProject(s, &project.Project, user, true, true)
if err != nil {
return
}
}
// Only using the default views
// Add all buckets to the default kanban view
for _, view := range project.Views {
if view.ViewKind == models.ProjectViewKindKanban {
for _, b := range bucketsByOldID {
b.ProjectViewID = view.ID
err = persistBucketView(b)
if err != nil {
return
}
}
break
}
}
}
log.Debugf("[creating structure] Creating %d tasks", len(tasks))
// Moving a done task into an imported bucket flips it back to open (it left the view's done bucket); restore after the loop in bulk.
now := time.Now()
taskIDsByDoneAt := make(map[time.Time][]int64)
setBucketOrDefault := func(task *models.Task) (err error) {
var bucketID = task.BucketID
bucket, exists := bucketsByOldID[bucketID]
if exists {
bucketID = bucket.ID
tb := &models.TaskBucket{
TaskID: task.ID,
BucketID: bucketID,
ProjectID: task.ProjectID,
ProjectViewID: bucket.ProjectViewID,
}
err = tb.Update(s, user)
if err != nil {
log.Debugf("[creating structure] Error while updating task bucket %d for task %d: %s", bucketID, task.ID, err.Error())
return
}
if task.Done {
doneAt := task.DoneAt
if doneAt.IsZero() {
doneAt = now
}
taskIDsByDoneAt[doneAt] = append(taskIDsByDoneAt[doneAt], task.ID)
}
} else if bucketID > 0 {
log.Debugf("[creating structure] No bucket created for original bucket id %d", task.BucketID)
bucketID = 0
}
if !exists || bucketID == 0 {
needsDefaultBucket = true
}
return
}
seedMissingTaskPositions(tasks)
type initialTask struct {
task *models.Task
comments []*models.TaskComment
bucketID int64
}
initialTasks := make([]initialTask, 0, len(tasks))
// Duplicate old ids resolve to the first queued struct.
canonicalByOldID := make(map[int64]*models.Task, len(tasks))
queue := func(task *models.Task, comments []*models.TaskComment) {
initialTasks = append(initialTasks, initialTask{
task: task,
comments: comments,
bucketID: task.BucketID,
})
if task.ID != 0 {
if _, exists := canonicalByOldID[task.ID]; !exists {
canonicalByOldID[task.ID] = task
}
}
}
for _, t := range tasks {
if t == nil || t.Title == "" {
continue
}
queue(&t.Task, t.Comments)
}
for i := 0; i < len(initialTasks); i++ {
for kind, relatedTasks := range initialTasks[i].task.RelatedTasks {
kept := make([]*models.Task, 0, len(relatedTasks))
for _, related := range relatedTasks {
if related == nil {
continue
}
// Id-only references (TickTick parents) must resolve before the title check.
if canonical, exists := canonicalByOldID[related.ID]; exists {
kept = append(kept, canonical)
continue
}
if related.Title == "" {
continue
}
queue(related, nil)
kept = append(kept, related)
}
initialTasks[i].task.RelatedTasks[kind] = kept
}
}
tasksToCreate := make([]*models.Task, 0, len(initialTasks))
for _, state := range initialTasks {
state.task.ProjectID = project.ID
state.task.BucketID = 0
state.task.Assignees = remapAssignees(state.task.Assignees, user)
tasksToCreate = append(tasksToCreate, state.task)
}
err = models.CreateTasksForImport(s, project.ID, tasksToCreate, user)
if err != nil {
return err
}
for _, state := range initialTasks {
t := state.task
t.BucketID = state.bucketID
err = setBucketOrDefault(t)
if err != nil {
return
}
log.Debugf("[creating structure] Created task %d", t.ID)
if len(t.RelatedTasks) > 0 {
log.Debugf("[creating structure] Creating %d related task kinds", len(t.RelatedTasks))
}
// Create all relation for each task
for kind, tasks := range t.RelatedTasks {
if len(tasks) > 0 {
log.Debugf("[creating structure] Creating %d related tasks for kind %v", len(tasks), kind)
}
for _, rt := range tasks {
taskRel := &models.TaskRelation{
TaskID: t.ID,
OtherTaskID: rt.ID,
RelationKind: kind,
}
// Add this check to prevent self-relations
if taskRel.TaskID == taskRel.OtherTaskID {
log.Debugf("[creating structure] Skipping invalid self-relation for task %d", taskRel.TaskID)
continue
}
err = taskRel.Create(s, user)
if err != nil && !models.IsErrRelationAlreadyExists(err) {
return
}
log.Debugf("[creating structure] Created task relation between task %d and %d", t.ID, rt.ID)
}
}
// Create all attachments for each task
if len(t.Attachments) > 0 {
log.Debugf("[creating structure] Creating %d attachments", len(t.Attachments))
}
for _, a := range t.Attachments {
if provider != nil {
err = createAttachmentFromProvider(s, t, a, user, provider, createdFiles)
if err != nil {
return
}
continue
}
// Check if we have a file to create
if len(a.File.FileContent) > 0 {
oldID := a.ID
a.ID = 0
a.TaskID = t.ID
// Import metadata is attacker-controlled and can forge a
// small size to bypass the file size limit (GHSA-qh78-rvg3-cv54).
actualSize := uint64(len(a.File.FileContent))
a.File.Size = actualSize
err = a.NewAttachment(s, bytes.NewReader(a.File.FileContent), a.File.Name, actualSize, user)
if err != nil {
if models.IsErrTaskAttachmentIsTooLarge(err) {
log.Warningf("[creating structure] Attachment %s is too large (%d bytes), skipping: %v", a.File.Name, actualSize, err)
continue
}
return
}
*createdFiles = append(*createdFiles, a.File.ID)
log.Debugf("[creating structure] Created new attachment %d", a.ID)
if t.CoverImageAttachmentID == oldID {
t.CoverImageAttachmentID = a.ID
err = t.Update(s, user)
if err != nil {
return
}
}
}
}
// Create all labels
for _, label := range t.Labels {
// Check if we already have a label with that name + color combination and use it
// If not, create one and save it for later
var lb *models.Label
var exists bool
if label == nil {
continue
}
key := label.Title + utils.NormalizeHex(label.HexColor)
lb, exists = labels[key]
if !exists {
err = label.Create(s, user)
if err != nil {
return err
}
log.Debugf("[creating structure] Created new label %d", label.ID)
labels[key] = label
lb = label
}
lt := &models.LabelTask{
LabelID: lb.ID,
TaskID: t.ID,
}
err = lt.Create(s, user)
if err != nil && !models.IsErrLabelIsAlreadyOnTask(err) {
return err
}
log.Debugf("[creating structure] Associated task %d with label %d", t.ID, lb.ID)
}
// Comments
for _, comment := range state.comments {
comment.TaskID = t.ID
comment.ID = 0
err = comment.CreateWithTimestamps(s, user)
if err != nil {
return
}
log.Debugf("[creating structure] Created new comment %d", comment.ID)
}
}
for doneAt, taskIDs := range taskIDsByDoneAt {
_, err = s.In("id", taskIDs).Cols("done", "done_at").Update(&models.Task{Done: true, DoneAt: doneAt})
if err != nil {
return
}
}
// All tasks brought their own bucket with them, therefore the newly created default buckets are just extra space.
// Delete all default-created buckets ("To-Do", "Doing", "Done") that were auto-generated.
if !needsDefaultBucket {
b := &models.Bucket{ProjectID: project.ID}
for _, view := range project.Views {
if view.ViewKind == models.ProjectViewKindKanban {
b.ProjectViewID = view.ID
break
}
}
bucketsIn, _, _, err := b.ReadAll(s, user, "", 1, 1)
if err != nil {
return err
}
buckets := bucketsIn.([]*models.Bucket)
migrationBucketIDs := make(map[int64]bool)
for _, mb := range bucketsByOldID {
migrationBucketIDs[mb.ID] = true
}
for _, b := range buckets {
if migrationBucketIDs[b.ID] {
continue
}
b.ProjectID = project.ID
err = b.Delete(s, user)
if err != nil && !models.IsErrCannotRemoveLastBucket(err) {
return err
}
}
}
if len(viewsByOldIDs) > 0 {
newPositions := []*models.TaskPosition{}
positionTaskIDs := make([]int64, 0, len(project.Positions))
for _, pos := range project.Positions {
_, hasTask := canonicalByOldID[pos.TaskID]
_, hasView := viewsByOldIDs[pos.ProjectViewID]
if !hasTask || !hasView {
continue
}
newPositions = append(newPositions, &models.TaskPosition{
TaskID: canonicalByOldID[pos.TaskID].ID,
ProjectViewID: viewsByOldIDs[pos.ProjectViewID].ID,
Position: pos.Position,
})
positionTaskIDs = append(positionTaskIDs, canonicalByOldID[pos.TaskID].ID)
}
if len(newPositions) > 0 {
_, err = s.In("task_id", positionTaskIDs).Delete(&models.TaskPosition{})
if err != nil {
return
}
_, err = s.Insert(newPositions)
if err != nil {
return
}
}
newTaskBuckets := make([]*models.TaskBucket, 0, len(project.TaskBuckets))
bucketTaskIDs := make([]int64, 0, len(project.TaskBuckets))
for _, tb := range project.TaskBuckets {
_, hasTask := canonicalByOldID[tb.TaskID]
_, hasBucket := bucketsByOldID[tb.BucketID]
if !hasTask || !hasBucket {
continue
}
newTaskBuckets = append(newTaskBuckets, &models.TaskBucket{
TaskID: canonicalByOldID[tb.TaskID].ID,
BucketID: bucketsByOldID[tb.BucketID].ID,
ProjectViewID: bucketsByOldID[tb.BucketID].ProjectViewID,
})
bucketTaskIDs = append(bucketTaskIDs, canonicalByOldID[tb.TaskID].ID)
}
if len(newTaskBuckets) > 0 {
_, err = s.In("task_id", bucketTaskIDs).Delete(&models.TaskBucket{})
if err != nil {
return
}
_, err = s.Insert(newTaskBuckets)
if err != nil {
return
}
}
}
project.Tasks = tasks
project.Buckets = originalBuckets
return nil
}
// Oversized provider files retain the preloaded path's skip behavior.
func createAttachmentFromProvider(s *xorm.Session, t *models.Task, a *models.TaskAttachment, user *user.User, provider FileProvider, createdFiles *[]int64) (err error) {
oldID := a.ID
content, size, err := provider.OpenAttachment(a)
if err != nil {
if files.IsErrFileIsTooLarge(err) {
log.Warningf("[creating structure] Attachment %s is too large, skipping: %v", a.File.Name, err)
return nil
}
return err
}
if content == nil {
return nil
}
defer func() {
_ = content.Close()
}()
a.ID = 0
a.TaskID = t.ID
// Import metadata can forge a smaller size than the stream (GHSA-qh78-rvg3-cv54).
a.File.Size = uint64(size) //nolint:gosec // size is bounded by the import budget
err = a.NewAttachment(s, content, a.File.Name, uint64(size), user)
if err != nil {
if models.IsErrTaskAttachmentIsTooLarge(err) {
log.Warningf("[creating structure] Attachment %s is too large (%d bytes), skipping: %v", a.File.Name, size, err)
return nil
}
return err
}
*createdFiles = append(*createdFiles, a.File.ID)
log.Debugf("[creating structure] Created new attachment %d", a.ID)
if t.CoverImageAttachmentID == oldID {
t.CoverImageAttachmentID = a.ID
err = t.Update(s, user)
if err != nil {
return err
}
}
return nil
}
// Foreign assignee IDs are invalid; only the importer can be matched by email or username.
func remapAssignees(assignees []*user.User, importer *user.User) []*user.User {
for _, a := range assignees {
if a == nil {
continue
}
emailMatch := a.Email != "" && importer.Email != "" && strings.EqualFold(a.Email, importer.Email)
usernameMatch := a.Username != "" && strings.EqualFold(a.Username, importer.Username)
if emailMatch || usernameMatch {
return []*user.User{importer}
}
}
return nil
}