vikunja/pkg/models/export.go

// Vikunja is a to-do list application to facilitate your life.
// Copyright 2018-present Vikunja and contributors. All rights reserved.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

package models

import (
	"archive/zip"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"os"
	"time"

	"code.vikunja.io/api/pkg/cron"
	"code.vikunja.io/api/pkg/db"
	"code.vikunja.io/api/pkg/files"
	"code.vikunja.io/api/pkg/log"
	"code.vikunja.io/api/pkg/notifications"
	"code.vikunja.io/api/pkg/user"
	"code.vikunja.io/api/pkg/utils"
	"code.vikunja.io/api/pkg/version"
	"code.vikunja.io/api/pkg/web"

	"xorm.io/builder"
	"xorm.io/xorm"
)

// The caller deletes the replaced export only after committing.
func ExportUserData(s *xorm.Session, u *user.User) (previousExportFileID int64, err error) {
	dumpFile, err := os.CreateTemp("", "vikunja-export-*.zip")
	if err != nil {
		return 0, fmt.Errorf("error creating temp file: %w", err)
	}
	tmpFilename := dumpFile.Name()
	defer os.Remove(tmpFilename) // #nosec G703 -- path from internally created temp file
	defer dumpFile.Close()

	dumpWriter := zip.NewWriter(dumpFile)
	defer dumpWriter.Close()

	// Get the data
	taskIDs, err := exportProjectsAndTasks(s, u, dumpWriter)
	if err != nil {
		return 0, err
	}
	// Task attachment files
	err = exportTaskAttachments(s, dumpWriter, taskIDs)
	if err != nil {
		return 0, err
	}
	// Saved filters
	err = exportSavedFilters(s, u, dumpWriter)
	if err != nil {
		return 0, err
	}
	// Background files
	err = exportProjectBackgrounds(s, u, dumpWriter)
	if err != nil {
		return 0, err
	}
	// Vikunja Version
	err = utils.WriteBytesToZip("VERSION", []byte(version.Version), dumpWriter)
	if err != nil {
		return 0, err
	}

	// If we reuse the same file again, saving it as a file in Vikunja will save it as a file with 0 bytes in size.
	// Closing and reopening does work.
	dumpWriter.Close()
	dumpFile.Close()

	exported, err := os.Open(tmpFilename) // #nosec G703 -- tmpFilename is generated internally, not from user input
	if err != nil {
		return 0, err
	}
	defer exported.Close()

	stat, err := exported.Stat()
	if err != nil {
		return 0, err
	}

	exportFile, err := files.CreateWithMimeAndSession(s, exported, tmpFilename, uint64(stat.Size()), u, "application/zip", false)
	if err != nil {
		return 0, err
	}
	defer func() {
		if err == nil {
			return
		}
		// The caller's rollback drops the row, not the blob.
		if rmErr := files.DeleteBlob(exportFile.ID); rmErr != nil && !errors.Is(rmErr, fs.ErrNotExist) {
			log.Errorf("Could not remove blob %d of failed export for user %d: %s", exportFile.ID, u.ID, rmErr)
		}
	}()

	current := &user.User{}
	_, err = s.Where("id = ?", u.ID).Cols("export_file_id").Get(current)
	if err != nil {
		return 0, err
	}

	err = replaceExportFileID(s, u.ID, current.ExportFileID, exportFile.ID)
	if err != nil {
		return 0, err
	}
	u.ExportFileID = exportFile.ID

	// Send a notification
	err = notifications.Notify(u, &DataExportReadyNotification{
		User: u,
	}, s)
	if err != nil {
		return 0, err
	}

	return current.ExportFileID, nil
}

// Fails if another export replaced oldID in the meantime, so the export it replaced isn't lost.
func replaceExportFileID(s *xorm.Session, userID, oldID, newID int64) error {
	var current builder.Cond = builder.Eq{"export_file_id": oldID}
	if oldID == 0 {
		current = builder.Or(current, builder.IsNull{"export_file_id"})
	}
	updated, err := s.
		Where("id = ?", userID).
		And(current).
		Cols("export_file_id").
		Update(&user.User{ExportFileID: newID})
	if err != nil {
		return err
	}
	if updated == 0 {
		return fmt.Errorf("export file of user %d changed during the export", userID)
	}
	return nil
}

// Only logs errors: the caller's change is already committed.
func deleteFileInNewSession(fileID int64) {
	if fileID == 0 {
		return
	}

	s := db.NewSession()
	defer s.Close()

	if err := deleteFileIfExists(s, fileID); err != nil {
		_ = s.Rollback()
		log.Errorf("Could not remove file %d: %s", fileID, err)
		return
	}
	if err := s.Commit(); err != nil {
		log.Errorf("Could not remove file %d: %s", fileID, err)
	}
}

func deleteFileIfExists(s *xorm.Session, fileID int64) error {
	if fileID == 0 {
		return nil
	}
	err := (&files.File{ID: fileID}).Delete(s)
	if err != nil && !files.IsErrFileDoesNotExist(err) {
		return err
	}
	return nil
}

func getRawTasksForExport(s *xorm.Session, projectIDs []int64, a web.Auth) (tasks []*Task, err error) {
	if len(projectIDs) == 0 {
		return []*Task{}, nil
	}

	opts := &taskSearchOptions{
		projectIDs: projectIDs,
		page:       0,
		perPage:    -1,
		sortby: []*sortParam{{
			sortBy:  taskPropertyID,
			orderBy: orderAscending,
		}},
	}

	hasFavoritesProject := false
	for _, id := range projectIDs {
		if id == FavoritesPseudoProject.ID {
			hasFavoritesProject = true
			break
		}
	}

	var dbSearcher taskSearcher = &dbTaskSearcher{
		s:                   s,
		a:                   a,
		hasFavoritesProject: hasFavoritesProject,
	}

	tasks, _, err = dbSearcher.Search(opts)
	if err != nil {
		return nil, err
	}

	taskMap := make(map[int64]*Task, len(tasks))
	for _, t := range tasks {
		taskMap[t.ID] = t
	}

	err = addMoreInfoToTasks(s, taskMap, a, nil, nil)
	if err != nil {
		return nil, err
	}

	return tasks, nil
}

type exportTaskReference struct {
	ID        int64 `json:"id"`
	ProjectID int64 `json:"project_id"`
}

// Related tasks are exported on their own; inlining them would repeat their content once per relation.
type exportTask struct {
	*TaskWithComments
	RelatedTasks map[RelationKind][]*exportTaskReference `json:"related_tasks"`
}

type exportProject struct {
	*ProjectWithTasksAndBuckets
	Tasks []*exportTask `json:"tasks"`
}

func toExportProjects(projects []*ProjectWithTasksAndBuckets) []*exportProject {
	exported := make([]*exportProject, 0, len(projects))
	for _, p := range projects {
		ep := &exportProject{ProjectWithTasksAndBuckets: p}
		for _, t := range p.Tasks {
			et := &exportTask{
				TaskWithComments: t,
				RelatedTasks:     make(map[RelationKind][]*exportTaskReference, len(t.RelatedTasks)),
			}
			for kind, related := range t.RelatedTasks {
				for _, rt := range related {
					et.RelatedTasks[kind] = append(et.RelatedTasks[kind], &exportTaskReference{
						ID:        rt.ID,
						ProjectID: rt.ProjectID,
					})
				}
			}
			ep.Tasks = append(ep.Tasks, et)
		}
		exported = append(exported, ep)
	}
	return exported
}

func exportProjectsAndTasks(s *xorm.Session, u *user.User, wr *zip.Writer) (taskIDs []int64, err error) {

	// Get all projects
	rawProjects, _, _, err := getRawProjectsForUser(
		s,
		&projectOptions{
			search:      "",
			user:        u,
			page:        0,
			perPage:     -1,
			getArchived: true,
		})
	if err != nil {
		return taskIDs, err
	}

	if len(rawProjects) == 0 {
		return
	}

	projects := []*ProjectWithTasksAndBuckets{}
	projectsMap := make(map[int64]*ProjectWithTasksAndBuckets, len(rawProjects))
	projectIDs := []int64{}
	for _, p := range rawProjects {
		if p.BackgroundFileID > 0 {
			p.BackgroundInformation = &files.File{
				ID: p.BackgroundFileID,
			}
		}
		pp := &ProjectWithTasksAndBuckets{
			Project: *p,
		}
		projects = append(projects, pp)
		projectsMap[p.ID] = pp
		projectIDs = append(projectIDs, p.ID)
	}

	views := map[int64]*ProjectView{}
	err = s.In("project_id", projectIDs).Find(&views)
	if err != nil {
		return
	}

	viewIDs := []int64{}
	for _, v := range views {
		if projectsMap[v.ProjectID].Views == nil {
			projectsMap[v.ProjectID].Views = []*ProjectView{}
		}
		projectsMap[v.ProjectID].Views = append(projectsMap[v.ProjectID].Views, v)
		viewIDs = append(viewIDs, v.ID)
	}

	tasks, err := getRawTasksForExport(s, projectIDs, u)
	if err != nil {
		return taskIDs, err
	}

	taskMap := make(map[int64]*TaskWithComments, len(tasks))
	for _, t := range tasks {
		taskMap[t.ID] = &TaskWithComments{
			Task: *t,
		}
		if _, exists := projectsMap[t.ProjectID]; !exists {
			log.Debugf("[User Data Export] Project %d does not exist for task %d, omitting", t.ProjectID, t.ID)
			continue
		}
		projectsMap[t.ProjectID].Tasks = append(projectsMap[t.ProjectID].Tasks, taskMap[t.ID])
		taskIDs = append(taskIDs, t.ID)
	}

	comments := []*TaskComment{}
	err = s.
		Join("LEFT", "tasks", "tasks.id = task_comments.task_id").
		In("tasks.project_id", projectIDs).
		And("tasks.deleted_at IS NULL").
		Find(&comments)
	if err != nil {
		return
	}

	for _, c := range comments {
		if _, exists := taskMap[c.TaskID]; !exists {
			log.Debugf("[User Data Export] Task %d does not exist for comment %d, omitting", c.TaskID, c.ID)
			continue
		}
		taskMap[c.TaskID].Comments = append(taskMap[c.TaskID].Comments, c)
	}

	buckets := []*Bucket{}
	err = s.In("project_view_id", viewIDs).Find(&buckets)
	if err != nil {
		return
	}

	bucketIDs := []int64{}
	for _, b := range buckets {
		view, exists := views[b.ProjectViewID]
		if !exists {
			log.Debugf("[User Data Export] Project view %d does not exist for bucket %d, omitting", b.ProjectViewID, b.ID)
			continue
		}
		_, exists = projectsMap[view.ProjectID]
		if !exists {
			log.Debugf("[User Data Export] Project %d does not exist for bucket %d, omitting", view.ProjectID, b.ID)
			continue
		}
		projectsMap[view.ProjectID].Buckets = append(projectsMap[view.ProjectID].Buckets, b)
		bucketIDs = append(bucketIDs, b.ID)
	}

	taskBuckets := []*TaskBucket{}
	err = s.In("bucket_id", bucketIDs).Find(&taskBuckets)
	if err != nil {
		return
	}

	for _, tb := range taskBuckets {
		view, exists := views[tb.ProjectViewID]
		if !exists {
			log.Debugf("[User Data Export] Project view %d does not exist, omitting", tb.ProjectViewID)
			continue
		}
		_, exists = projectsMap[view.ProjectID]
		if !exists {
			log.Debugf("[User Data Export] Project %d does not exist, omitting", view.ProjectID)
			continue
		}

		if projectsMap[view.ProjectID].TaskBuckets == nil {
			projectsMap[view.ProjectID].TaskBuckets = []*TaskBucket{}
		}

		projectsMap[view.ProjectID].TaskBuckets = append(projectsMap[view.ProjectID].TaskBuckets, tb)
	}

	taskPositions := []*TaskPosition{}
	err = s.In("project_view_id", viewIDs).Find(&taskPositions)
	if err != nil {
		return
	}

	for _, p := range taskPositions {
		view, exists := views[p.ProjectViewID]
		if !exists {
			log.Debugf("[User Data Export] Project view %d does not exist, omitting", p.ProjectViewID)
			continue
		}
		_, exists = projectsMap[view.ProjectID]
		if !exists {
			log.Debugf("[User Data Export] Project %d does not exist, omitting", view.ProjectID)
			continue
		}

		if projectsMap[view.ProjectID].Positions == nil {
			projectsMap[view.ProjectID].Positions = []*TaskPosition{}
		}

		projectsMap[view.ProjectID].Positions = append(projectsMap[view.ProjectID].Positions, p)
	}

	data, err := json.Marshal(toExportProjects(projects))
	if err != nil {
		return taskIDs, err
	}

	return taskIDs, utils.WriteBytesToZip("data.json", data, wr)
}

func exportTaskAttachments(s *xorm.Session, wr *zip.Writer, taskIDs []int64) (err error) {
	tas, err := getTaskAttachmentsByTaskIDs(s, taskIDs)
	if err != nil {
		return err
	}

	attachmentFiles := make(map[int64]io.ReadCloser)
	for _, ta := range tas {
		err = ta.File.LoadFileByID()
		if err != nil {
			if files.IsErrFileDoesNotExist(err) {
				continue
			}
			return err
		}
		attachmentFiles[ta.FileID] = ta.File.File
	}

	return utils.WriteFilesToZip(attachmentFiles, wr)
}

func exportSavedFilters(s *xorm.Session, u *user.User, wr *zip.Writer) (err error) {
	filters, err := getSavedFiltersForUser(s, u, "")
	if err != nil {
		return err
	}

	data, err := json.Marshal(filters)
	if err != nil {
		return err
	}

	return utils.WriteBytesToZip("filters.json", data, wr)
}

func exportProjectBackgrounds(s *xorm.Session, u *user.User, wr *zip.Writer) (err error) {
	projects, _, _, err := getRawProjectsForUser(
		s,
		&projectOptions{
			user: u,
			page: -1,
		},
	)
	if err != nil {
		return err
	}

	backgroundFiles := make(map[int64]io.ReadCloser)
	for _, l := range projects {
		if l.BackgroundFileID == 0 {
			continue
		}

		bgFile := &files.File{
			ID: l.BackgroundFileID,
		}
		err = bgFile.LoadFileByID()
		if err != nil {
			if files.IsErrFileDoesNotExist(err) {
				continue
			}
			return err
		}

		backgroundFiles[l.BackgroundFileID] = bgFile.File
	}

	return utils.WriteFilesToZip(backgroundFiles, wr)
}

// GetUserDataExportFile loads the export's metadata; OpenUserDataExportFile reads the contents.
func GetUserDataExportFile(s *xorm.Session, u *user.User) (*files.File, error) {
	if u.ExportFileID == 0 {
		return nil, ErrUserDataExportDoesNotExist{}
	}

	exportFile := &files.File{ID: u.ExportFileID}
	if err := exportFile.LoadFileMetaByID(s); err != nil {
		if files.IsErrFileDoesNotExist(err) {
			return nil, ErrUserDataExportDoesNotExist{}
		}
		return nil, err
	}

	return exportFile, nil
}

// OpenUserDataExportFile hits object storage, so callers must commit their db
// session first; the caller must close the reader.
func OpenUserDataExportFile(exportFile *files.File) error {
	if err := exportFile.LoadFileByID(); err != nil {
		if files.IsErrFileDoesNotExist(err) {
			return ErrUserDataExportDoesNotExist{}
		}
		return err
	}
	return nil
}

// GetUserDataExportStatus returns metadata about the user's current data export, or
// nil when none exists. The expiry mirrors the cleanup cron's 7-day retention.
func GetUserDataExportStatus(s *xorm.Session, u *user.User) (*UserExportStatus, error) {
	if u.ExportFileID == 0 {
		return nil, nil
	}

	exportFile := &files.File{ID: u.ExportFileID}
	if err := exportFile.LoadFileMetaByID(s); err != nil {
		// A missing meta row means there is no export — mirror the download path
		// (404 there) instead of surfacing a 500.
		if files.IsErrFileDoesNotExist(err) {
			return nil, nil
		}
		return nil, err
	}

	return &UserExportStatus{
		ID:      exportFile.ID,
		Size:    exportFile.Size,
		Created: exportFile.Created,
		Expires: exportFile.Created.Add(7 * 24 * time.Hour),
	}, nil
}

// UserExportStatus is the metadata returned for a user's current data export.
type UserExportStatus struct {
	ID      int64     `json:"id" readOnly:"true" doc:"The id of the export file."`
	Size    uint64    `json:"size" readOnly:"true" doc:"The size of the export file in bytes."`
	Created time.Time `json:"created" readOnly:"true" doc:"When the export was created."`
	Expires time.Time `json:"expires" readOnly:"true" doc:"When the export will be automatically deleted (7 days after creation)."`
}

func RegisterOldExportCleanupCron() {
	err := cron.Schedule("0 * * * *", cleanupOldUserExports)
	if err != nil {
		log.Fatalf("Could not register old export cleanup cron: %s", err)
	}
}

func cleanupOldUserExports() {
	const logPrefix = "[User Export Cleanup Cron] "

	s := db.NewSession()
	defer s.Close()

	users := []*user.User{}
	err := s.Where("export_file_id IS NOT NULL AND export_file_id != ?", 0).Find(&users)
	if err != nil {
		log.Errorf(logPrefix+"Could not get users with export files: %s", err)
		return
	}

	fileIDs := []int64{}
	for _, u := range users {
		fileIDs = append(fileIDs, u.ExportFileID)
	}

	fs := []*files.File{}
	err = s.Where("created < ?", time.Now().Add(-time.Hour*24*7)).In("id", fileIDs).Find(&fs)
	if err != nil {
		log.Errorf(logPrefix+"Could not get users with export files: %s", err)
		return
	}

	if len(fs) == 0 {
		return
	}

	log.Debugf(logPrefix+"Removing %d old user data exports...", len(fs))

	expiredIDs := make([]int64, 0, len(fs))
	for _, f := range fs {
		expiredIDs = append(expiredIDs, f.ID)
		err = deleteFileIfExists(s, f.ID)
		if err != nil {
			log.Errorf(logPrefix+"Could not remove user export file %d: %s", f.ID, err)
			return
		}
	}

	_, err = s.In("export_file_id", expiredIDs).Cols("export_file_id").Update(&user.User{})
	if err != nil {
		log.Errorf(logPrefix+"Could not update user export file state: %s", err)
		return
	}

	log.Debugf(logPrefix+"Removed %d old user data exports...", len(fs))

	if err := s.Commit(); err != nil {
		log.Errorf(logPrefix+"Error committing export cleanup: %s", err)
	}
}