feat(core): persist tasks and recover them after restart

database/db.go

@@ -35,7 +35,7 @@ func Init(ctx context.Context) {
 		logger.Fatal("Failed to open database: ", err)
 	}
 	logger.Debug("Database connected")
-	if err := db.AutoMigrate(&User{}, &Dir{}, &Rule{}, &WatchChat{}); err != nil {
+	if err := db.AutoMigrate(&User{}, &Dir{}, &Rule{}, &WatchChat{}, &Task{}); err != nil {
 		logger.Fatal("Database migration failed; if upgrading from an old version, try deleting the database file and retrying", "error", err)
 	}
 	if err := syncUsers(ctx); err != nil {

database/task.go

@@ -0,0 +1,92 @@
+package database
+
+import (
+	"context"
+	"errors"
+	"time"
+)
+
+var errNotInitialized = errors.New("database not initialized")
+
+type TaskStatus string
+
+const (
+	TaskStatusQueued    TaskStatus = "queued"
+	TaskStatusRunning   TaskStatus = "running"
+	TaskStatusFailed    TaskStatus = "failed"
+	TaskStatusCancelled TaskStatus = "cancelled"
+)
+
+// Task is the persisted record of a queued or running task, used to recover
+// unfinished work after a process restart. Completed tasks are deleted on
+// finish, so the table only ever holds queued/running rows.
+type Task struct {
+	ID        string `gorm:"primaryKey;size:64"`
+	Type      string `gorm:"size:32;index"`
+	Payload   []byte
+	Status    string `gorm:"size:16;index"`
+	Error     string
+	CreatedAt time.Time
+	UpdatedAt time.Time
+}
+
+func CreateTask(ctx context.Context, task *Task) error {
+	if db == nil {
+		return errNotInitialized
+	}
+	return db.WithContext(ctx).Create(task).Error
+}
+
+// UpsertTask inserts the task or replaces the existing row with the same ID.
+func UpsertTask(ctx context.Context, task *Task) error {
+	if db == nil {
+		return errNotInitialized
+	}
+	return db.WithContext(ctx).Save(task).Error
+}
+
+func UpdateTaskStatus(ctx context.Context, id string, status TaskStatus, errMsg string) error {
+	if db == nil {
+		return errNotInitialized
+	}
+	return db.WithContext(ctx).Model(&Task{}).
+		Where("id = ?", id).
+		Updates(map[string]any{
+			"status":     status,
+			"error":      errMsg,
+			"updated_at": time.Now(),
+		}).Error
+}
+
+func DeleteTask(ctx context.Context, id string) error {
+	if db == nil {
+		return errNotInitialized
+	}
+	return db.WithContext(ctx).Delete(&Task{}, "id = ?", id).Error
+}
+
+// GetUnfinishedTasks returns all tasks that were not finished when the
+// process stopped, i.e. tasks that must be re-enqueued on startup.
+func GetUnfinishedTasks(ctx context.Context) ([]Task, error) {
+	if db == nil {
+		return nil, errNotInitialized
+	}
+	var tasks []Task
+	err := db.WithContext(ctx).
+		Where("status IN ?", []string{string(TaskStatusQueued), string(TaskStatusRunning)}).
+		Order("created_at").
+		Find(&tasks).Error
+	return tasks, err
+}
+
+func CountUnfinishedTasks(ctx context.Context) (int64, error) {
+	if db == nil {
+		return 0, errNotInitialized
+	}
+	var count int64
+	err := db.WithContext(ctx).
+		Model(&Task{}).
+		Where("status IN ?", []string{string(TaskStatusQueued), string(TaskStatusRunning)}).
+		Count(&count).Error
+	return count, err
+}

database/task_test.go

@@ -0,0 +1,110 @@
+package database
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+
+	"github.com/ncruces/go-sqlite3/gormlite"
+	"gorm.io/gorm"
+)
+
+func newTestDB(t *testing.T) {
+	t.Helper()
+	d, err := gorm.Open(gormlite.Open(filepath.Join(t.TempDir(), "test.db")), &gorm.Config{})
+	if err != nil {
+		t.Fatalf("open test db: %v", err)
+	}
+	if err := d.AutoMigrate(&Task{}); err != nil {
+		t.Fatalf("migrate: %v", err)
+	}
+	old := db
+	db = d
+	t.Cleanup(func() { db = old })
+}
+
+func TestTaskCRUD(t *testing.T) {
+	newTestDB(t)
+	ctx := context.Background()
+
+	task := &Task{
+		ID:      "task-1",
+		Type:    "tfile",
+		Payload: []byte(`{"file":"x"}`),
+		Status:  string(TaskStatusQueued),
+	}
+	if err := CreateTask(ctx, task); err != nil {
+		t.Fatalf("create: %v", err)
+	}
+
+	unfinished, err := GetUnfinishedTasks(ctx)
+	if err != nil {
+		t.Fatalf("get unfinished: %v", err)
+	}
+	if len(unfinished) != 1 || unfinished[0].ID != "task-1" {
+		t.Fatalf("got %+v, want 1 task task-1", unfinished)
+	}
+
+	if err := UpdateTaskStatus(ctx, "task-1", TaskStatusRunning, ""); err != nil {
+		t.Fatalf("update: %v", err)
+	}
+	unfinished, err = GetUnfinishedTasks(ctx)
+	if err != nil {
+		t.Fatalf("get unfinished after update: %v", err)
+	}
+	if len(unfinished) != 1 || unfinished[0].Status != string(TaskStatusRunning) {
+		t.Fatalf("running status not persisted: %+v", unfinished)
+	}
+
+	if err := DeleteTask(ctx, "task-1"); err != nil {
+		t.Fatalf("delete: %v", err)
+	}
+	count, err := CountUnfinishedTasks(ctx)
+	if err != nil {
+		t.Fatalf("count: %v", err)
+	}
+	if count != 0 {
+		t.Fatalf("count = %d, want 0", count)
+	}
+}
+
+func TestTaskUpsert(t *testing.T) {
+	newTestDB(t)
+	ctx := context.Background()
+
+	task := &Task{ID: "task-2", Type: "tfile", Status: string(TaskStatusQueued)}
+	if err := UpsertTask(ctx, task); err != nil {
+		t.Fatalf("upsert create: %v", err)
+	}
+	task.Status = string(TaskStatusRunning)
+	task.Payload = []byte("new")
+	if err := UpsertTask(ctx, task); err != nil {
+		t.Fatalf("upsert update: %v", err)
+	}
+	unfinished, err := GetUnfinishedTasks(ctx)
+	if err != nil {
+		t.Fatalf("get unfinished: %v", err)
+	}
+	if len(unfinished) != 1 || unfinished[0].Status != string(TaskStatusRunning) || string(unfinished[0].Payload) != "new" {
+		t.Fatalf("upsert did not replace: %+v", unfinished)
+	}
+}
+
+func TestGetUnfinishedTasksExcludesFinished(t *testing.T) {
+	newTestDB(t)
+	ctx := context.Background()
+
+	if err := CreateTask(ctx, &Task{ID: "done", Type: "tfile", Status: string(TaskStatusFailed)}); err != nil {
+		t.Fatal(err)
+	}
+	if err := CreateTask(ctx, &Task{ID: "pending", Type: "tfile", Status: string(TaskStatusQueued)}); err != nil {
+		t.Fatal(err)
+	}
+	unfinished, err := GetUnfinishedTasks(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(unfinished) != 1 || unfinished[0].ID != "pending" {
+		t.Fatalf("got %+v, want only pending", unfinished)
+	}
+}