diff --git a/internal/syncjob/continuous_backoff.go b/internal/syncjob/continuous_backoff.go new file mode 100644 index 00000000..fbc98773 --- /dev/null +++ b/internal/syncjob/continuous_backoff.go @@ -0,0 +1,78 @@ +package syncjob + +import ( + "context" + "hash/fnv" + "time" +) + +const ( + continuousFailureBackoffInitial = 5 * time.Second + continuousFailureBackoffMaximum = 5 * time.Minute + continuousFailureHistoryLimit = 32 +) + +func (m *Manager) continuousFailureNotBefore(ctx context.Context, definition JobDefinition) (int64, int, error) { + if definition.Schedule.Kind != ScheduleContinuous { + return 0, 0, nil + } + runs, err := m.store.ListRuns(ctx, definition.ID, continuousFailureHistoryLimit) + if err != nil { + return 0, 0, err + } + consecutiveFailures := 0 + var latestFailure RunRecord + for _, run := range runs { + switch run.Status { + case RunStatusQueued, RunStatusRunning, RunStatusCancelling: + continue + case RunStatusFailed, RunStatusPartial, RunStatusInterrupted: + if consecutiveFailures == 0 { + latestFailure = run + } + consecutiveFailures++ + default: + // A success or an operator-controlled terminal state ends the failure + // streak. The next continuous launch returns to the normal poll cadence. + goto counted + } + } + +counted: + if consecutiveFailures == 0 || latestFailure.FinishedAt <= 0 { + return 0, consecutiveFailures, nil + } + backoff := continuousFailureBackoff(definition.ID, latestFailure.ID, consecutiveFailures) + return time.UnixMilli(latestFailure.FinishedAt).Add(backoff).UnixMilli(), consecutiveFailures, nil +} + +func continuousFailureBackoff(jobID, latestRunID string, consecutiveFailures int) time.Duration { + if consecutiveFailures < 1 { + return 0 + } + base := continuousFailureBackoffInitial + for attempt := 1; attempt < consecutiveFailures && base < continuousFailureBackoffMaximum; attempt++ { + if base > continuousFailureBackoffMaximum/2 { + base = continuousFailureBackoffMaximum + break + } + base *= 2 + } + if base >= continuousFailureBackoffMaximum { + return continuousFailureBackoffMaximum + } + jitterRoom := base / 5 + if remaining := continuousFailureBackoffMaximum - base; jitterRoom > remaining { + jitterRoom = remaining + } + if jitterRoom <= 0 { + return base + } + hasher := fnv.New64a() + _, _ = hasher.Write([]byte(jobID)) + _, _ = hasher.Write([]byte{0}) + _, _ = hasher.Write([]byte(latestRunID)) + _, _ = hasher.Write([]byte{0, byte(consecutiveFailures)}) + jitter := time.Duration(hasher.Sum64() % (uint64(jitterRoom) + 1)) + return base + jitter +} diff --git a/internal/syncjob/continuous_backoff_test.go b/internal/syncjob/continuous_backoff_test.go new file mode 100644 index 00000000..05d07f62 --- /dev/null +++ b/internal/syncjob/continuous_backoff_test.go @@ -0,0 +1,169 @@ +package syncjob + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" +) + +func TestContinuousFailureBackoffPersistsAcrossRestartAndSuccessResets(t *testing.T) { + path := t.TempDir() + "/continuous.db" + store, err := Open(path) + if err != nil { + t.Fatalf("open store: %v", err) + } + definition := putContinuousTestJob(t, store) + failedAt := time.Now().Truncate(time.Millisecond) + createTerminalHistoryRun(t, store, definition, RunStatusFailed, failedAt) + now := failedAt.Add(time.Second) + if _, err := store.db.ExecContext(context.Background(), `UPDATE data_sync_jobs SET next_run_at = ? WHERE id = ?`, now.Add(-time.Millisecond).UnixMilli(), definition.ID); err != nil { + t.Fatalf("make continuous job due: %v", err) + } + manager := newManualSchedulerManager(store, now, "first-owner") + notBefore, failures, err := manager.continuousFailureNotBefore(context.Background(), definition) + if err != nil || failures != 1 { + t.Fatalf("failure backoff = %d, failures=%d, err=%v", notBefore, failures, err) + } + delay := time.Duration(notBefore-failedAt.UnixMilli()) * time.Millisecond + if delay < 5*time.Second || delay > 6*time.Second { + t.Fatalf("first failure delay = %s, want [5s, 6s]", delay) + } + manager.runSchedulerCycle() + delayed, err := store.GetJob(context.Background(), definition.ID) + if err != nil || delayed.NextRunAt != notBefore { + t.Fatalf("delayed job = %#v, err=%v", delayed, err) + } + runs, err := store.ListRuns(context.Background(), definition.ID, 10) + if err != nil || len(runs) != 1 { + t.Fatalf("runs during backoff = %#v, err=%v", runs, err) + } + _ = store.ReleaseSchedulerLease(context.Background(), "data-sync-scheduler", "first-owner") + if err := store.Close(); err != nil { + t.Fatalf("close store: %v", err) + } + + reopened, err := Open(path) + if err != nil { + t.Fatalf("reopen store: %v", err) + } + t.Cleanup(func() { _ = reopened.Close() }) + restarted := newManualSchedulerManager(reopened, now, "restart-owner") + reloaded, err := reopened.GetJob(context.Background(), definition.ID) + if err != nil || reloaded.NextRunAt != notBefore { + t.Fatalf("reloaded delayed job = %#v, err=%v", reloaded, err) + } + restartedNotBefore, restartedFailures, err := restarted.continuousFailureNotBefore(context.Background(), reloaded) + if err != nil || restartedFailures != failures || restartedNotBefore != notBefore { + t.Fatalf("restart backoff = %d/%d, want %d/%d, err=%v", restartedNotBefore, restartedFailures, notBefore, failures, err) + } + restarted.runSchedulerCycle() + runs, err = reopened.ListRuns(context.Background(), definition.ID, 10) + if err != nil || len(runs) != 1 { + t.Fatalf("restart runs during backoff = %#v, err=%v", runs, err) + } + + createTerminalHistoryRun(t, reopened, reloaded, RunStatusSucceeded, failedAt.Add(2*time.Second)) + resetAt, resetFailures, err := restarted.continuousFailureNotBefore(context.Background(), reloaded) + if err != nil || resetAt != 0 || resetFailures != 0 { + t.Fatalf("success reset backoff = %d, failures=%d, err=%v", resetAt, resetFailures, err) + } +} + +func TestContinuousFailureBackoffIsStableExponentialAndCapped(t *testing.T) { + previous := time.Duration(0) + for failures := 1; failures <= 12; failures++ { + first := continuousFailureBackoff("job", "run", failures) + second := continuousFailureBackoff("job", "run", failures) + if first != second { + t.Fatalf("failure %d jitter is not stable: %s != %s", failures, first, second) + } + if first < previous || first > 5*time.Minute { + t.Fatalf("failure %d backoff = %s, previous=%s", failures, first, previous) + } + previous = first + } + if previous != 5*time.Minute { + t.Fatalf("capped backoff = %s, want 5m", previous) + } +} + +func TestPermanentExecutionErrorPausesOwningJob(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "queue") + manager := newTestManager(t, store, ExecutorFunc(func(context.Context, ExecutionRequest, RunReporter) (ExecutionOutcome, error) { + return ExecutionOutcome{}, MarkPermanentExecutionError(errors.New("unsupported source topology")) + })) + run, err := manager.StartRun(context.Background(), definition.ID) + if err != nil { + t.Fatalf("start run: %v", err) + } + waitRunStatus(t, store, run.ID, RunStatusFailed) + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + paused, getErr := store.GetJob(context.Background(), definition.ID) + if getErr == nil && paused.Lifecycle == JobLifecyclePaused && !paused.Enabled { + if _, err := manager.StartRun(context.Background(), definition.ID); !errors.Is(err, ErrJobDisabled) { + t.Fatalf("start paused job error = %v, want ErrJobDisabled", err) + } + return + } + time.Sleep(time.Millisecond) + } + t.Fatal("permanent failure did not pause job") +} + +func putContinuousTestJob(t *testing.T, store *Store) JobDefinition { + t.Helper() + definition := JobDefinition{ + Name: "continuous orders", Lifecycle: JobLifecycleEnabled, Enabled: true, Kind: JobKindReconcile, + IncrementalMode: IncrementalCDC, Source: EndpointRef{ConnectionID: "source"}, Target: EndpointRef{ConnectionID: "target"}, + Mappings: []TableMapping{{SourceTable: "orders", TargetTable: "orders", KeyColumns: []string{"id"}, Enabled: true}}, + CDC: &CDCSpec{Adapter: "mongodb-change-stream", StartPosition: "checkpoint"}, + Schedule: ScheduleSpec{Kind: ScheduleContinuous}, ConcurrencyPolicy: "forbid", + } + saved, err := store.PutJob(context.Background(), definition) + if err != nil { + t.Fatalf("put continuous job: %v", err) + } + return saved +} + +func createTerminalHistoryRun(t *testing.T, store *Store, definition JobDefinition, status RunStatus, finishedAt time.Time) RunRecord { + t.Helper() + snapshot, err := json.Marshal(definition) + if err != nil { + t.Fatalf("marshal definition: %v", err) + } + run, err := store.CreateRun(context.Background(), RunRecord{ + JobID: definition.ID, JobRevision: definition.Revision, Status: RunStatusRunning, DefinitionSnapshot: snapshot, + }) + if err != nil { + t.Fatalf("create history run: %v", err) + } + run, err = store.CompleteRun(context.Background(), run.ID, status, ExecutionOutcome{}, string(status), finishedAt.UnixMilli()) + if err != nil { + t.Fatalf("complete history run: %v", err) + } + if _, err := store.db.ExecContext(context.Background(), `UPDATE data_sync_runs SET created_at = ?, updated_at = ? WHERE id = ?`, + finishedAt.UnixMilli(), finishedAt.UnixMilli(), run.ID); err != nil { + t.Fatalf("order history run: %v", err) + } + return run +} + +func newManualSchedulerManager(store *Store, now time.Time, owner string) *Manager { + ctx, cancel := context.WithCancelCause(context.Background()) + options := normalizeManagerOptions(ManagerOptions{ + SchedulerInterval: time.Hour, LeaseTTL: time.Minute, HeartbeatInterval: time.Hour, + RecoveryStaleAfter: time.Hour, RecoveryInterval: time.Hour, LeaseOwner: owner, Now: func() time.Time { return now }, + }) + return &Manager{ + store: store, executor: ExecutorFunc(func(context.Context, ExecutionRequest, RunReporter) (ExecutionOutcome, error) { + return ExecutionOutcome{}, nil + }), + options: options, ctx: ctx, cancel: cancel, wake: make(chan struct{}, 1), active: make(map[string]activeExecution), + lastRecoveryAt: now, done: make(chan struct{}), + } +} diff --git a/internal/syncjob/manager.go b/internal/syncjob/manager.go new file mode 100644 index 00000000..551ca51c --- /dev/null +++ b/internal/syncjob/manager.go @@ -0,0 +1,1159 @@ +package syncjob + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "time" + + "github.com/google/uuid" +) + +var ( + ErrManagerClosed = errors.New("data sync job manager is closed") + ErrJobDisabled = errors.New("data sync job is disabled") + ErrRunNotResumable = errors.New("data sync run is not resumable") + ErrRunNotRetryable = errors.New("data sync run is not retryable") + errManagerShutdown = errors.New("data sync job manager is shutting down") + errRunCanceled = errors.New("data sync run cancellation requested") +) + +const ( + errorRowRetryLeaseTTL = 30 * time.Second + errorRowRetryRenewInterval = 10 * time.Second + errorRowRetryFinalizeTTL = 5 * time.Second +) + +type ExecutionRequest struct { + Run RunRecord `json:"run"` + Definition JobDefinition `json:"definition"` + Checkpoint *Checkpoint `json:"checkpoint,omitempty"` +} + +type RunReporter interface { + ReportProgress(RunProgress) error + SaveCheckpoint(Checkpoint) error + AppendErrorRow(ErrorRow) error + Emit(RunEventType, string, json.RawMessage) error +} + +type Executor interface { + Execute(context.Context, ExecutionRequest, RunReporter) (ExecutionOutcome, error) +} + +type ExecutorFunc func(context.Context, ExecutionRequest, RunReporter) (ExecutionOutcome, error) + +func (execute ExecutorFunc) Execute(ctx context.Context, request ExecutionRequest, reporter RunReporter) (ExecutionOutcome, error) { + return execute(ctx, request, reporter) +} + +type ManagerHooks struct { + OnRunEvent func(RunEvent) +} + +type ManagerOptions struct { + SchedulerInterval time.Duration + LeaseTTL time.Duration + HeartbeatInterval time.Duration + RecoveryStaleAfter time.Duration + RecoveryInterval time.Duration + MaxConcurrentRuns int + LeaseOwner string + Hooks ManagerHooks + Now func() time.Time +} + +type Manager struct { + store *Store + executor Executor + options ManagerOptions + + ctx context.Context + cancel context.CancelCauseFunc + wake chan struct{} + + mu sync.Mutex + closing bool + active map[string]activeExecution + wg sync.WaitGroup + + lastRecoveryAt time.Time + + shutdownOnce sync.Once + done chan struct{} +} + +type activeExecution struct { + jobID string + ownerToken string + cancel context.CancelCauseFunc +} + +func NewManager(ctx context.Context, store *Store, executor Executor, options ManagerOptions) (*Manager, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return nil, err + } + if store == nil { + return nil, errors.New("data sync job store is required") + } + if err := store.ensureOpen(); err != nil { + return nil, err + } + if executor == nil { + return nil, errors.New("data sync job executor is required") + } + options = normalizeManagerOptions(options) + managerCtx, cancel := context.WithCancelCause(context.Background()) + manager := &Manager{ + store: store, + executor: executor, + options: options, + ctx: managerCtx, + cancel: cancel, + wake: make(chan struct{}, 1), + active: make(map[string]activeExecution), + done: make(chan struct{}), + } + now := options.Now() + acquired, err := store.AcquireSchedulerLease(ctx, "data-sync-scheduler", options.LeaseOwner, now, options.LeaseTTL) + if err != nil { + cancel(err) + return nil, err + } + if acquired { + if err := manager.recoverInterrupted(ctx); err != nil { + _ = store.ReleaseSchedulerLease(context.Background(), "data-sync-scheduler", options.LeaseOwner) + cancel(err) + return nil, err + } + manager.lastRecoveryAt = now + } + manager.wg.Add(2) + go manager.dispatchLoop() + go manager.schedulerLoop() + manager.signalWake() + return manager, nil +} + +func (m *Manager) recoverInterrupted(ctx context.Context) error { + now := m.options.Now() + if _, err := m.store.RecoverExpiredErrorRowRetries(ctx, now.UnixMilli()); err != nil { + return err + } + recovered, err := m.store.InterruptStaleRuns(ctx, now.Add(-m.options.RecoveryStaleAfter).UnixMilli(), now.UnixMilli()) + if err != nil { + return err + } + for _, run := range recovered { + if run.Status == RunStatusCanceled { + if _, err := m.appendEvent(ctx, run, RunEventCanceled, "canceled after manager restart", nil); err != nil { + return err + } + continue + } + if _, err := m.appendEvent(ctx, run, RunEventInterrupted, "interrupted after manager restart", nil); err != nil { + return err + } + definition, decodeErr := decodeRunDefinition(run) + if decodeErr != nil || definition.ResumePolicy != "auto" { + continue + } + if _, resumeErr := m.ResumeRun(ctx, run.ID); resumeErr != nil { + payload, _ := json.Marshal(map[string]string{"error": resumeErr.Error()}) + _, _ = m.appendEvent(ctx, run, RunEventLog, "automatic resume was not queued", payload) + } + } + return nil +} + +func normalizeManagerOptions(options ManagerOptions) ManagerOptions { + if options.SchedulerInterval <= 0 { + options.SchedulerInterval = time.Second + } + if options.LeaseTTL <= 0 { + options.LeaseTTL = 10 * time.Second + } + if options.HeartbeatInterval <= 0 { + options.HeartbeatInterval = 5 * time.Second + } + if options.RecoveryStaleAfter <= 0 { + options.RecoveryStaleAfter = 3 * options.HeartbeatInterval + } + if options.RecoveryInterval <= 0 { + options.RecoveryInterval = options.RecoveryStaleAfter + } + if options.MaxConcurrentRuns <= 0 { + options.MaxConcurrentRuns = 4 + } + if options.Now == nil { + options.Now = time.Now + } + if strings.TrimSpace(options.LeaseOwner) == "" { + options.LeaseOwner = "sync-scheduler-" + uuid.NewString() + } + return options +} + +func (m *Manager) PutJob(ctx context.Context, definition JobDefinition) (JobDefinition, error) { + if err := m.ensureOpen(); err != nil { + return JobDefinition{}, err + } + saved, err := m.store.PutJob(ctx, definition) + if err != nil { + return JobDefinition{}, err + } + if saved.Lifecycle == JobLifecyclePaused || saved.Lifecycle == JobLifecycleArchived { + m.cancelLocalJobRuns(saved.ID, errRunCanceled) + m.signalWake() + } + return saved, nil +} + +func (m *Manager) PauseJob(ctx context.Context, id string) (JobDefinition, error) { + if err := m.ensureOpen(); err != nil { + return JobDefinition{}, err + } + paused, err := m.store.PauseJob(ctx, strings.TrimSpace(id)) + if err != nil { + return JobDefinition{}, err + } + m.cancelLocalJobRuns(paused.ID, errRunCanceled) + m.signalWake() + return paused, nil +} + +func (m *Manager) GetJob(ctx context.Context, id string) (JobDefinition, error) { + if err := m.ensureOpen(); err != nil { + return JobDefinition{}, err + } + return m.store.GetJob(ctx, id) +} + +func (m *Manager) ListJobs(ctx context.Context) ([]JobDefinition, error) { + if err := m.ensureOpen(); err != nil { + return nil, err + } + return m.store.ListJobs(ctx) +} + +func (m *Manager) DeleteJob(ctx context.Context, id string) error { + if err := m.ensureOpen(); err != nil { + return err + } + id = strings.TrimSpace(id) + transitions, err := m.store.archiveJobAndCancelRuns(ctx, id, m.nowMillis()) + if err != nil { + return err + } + for _, transition := range transitions { + eventType := RunEventCancelling + message := "cancellation requested because task was archived" + if transition.Run.Status == RunStatusCanceled { + eventType = RunEventCanceled + message = "canceled because task was archived" + } + _, _ = m.appendEvent(ctx, transition.Run, eventType, message, nil) + } + m.cancelLocalJobRuns(id, errRunCanceled) + m.signalWake() + return nil +} + +func (m *Manager) GetRun(ctx context.Context, id string) (RunRecord, error) { + return m.store.GetRun(ctx, id) +} + +func (m *Manager) ListRuns(ctx context.Context, jobID string, limit int) ([]RunRecord, error) { + return m.store.ListRuns(ctx, jobID, limit) +} + +func (m *Manager) ListRunEvents(ctx context.Context, runID string, afterSequence int64, limit int) ([]RunEvent, error) { + return m.store.ListRunEvents(ctx, runID, afterSequence, limit) +} + +func (m *Manager) ListErrorRows(ctx context.Context, runID string, status ErrorRowStatus, limit int) ([]ErrorRow, error) { + return m.store.ListErrorRows(ctx, runID, status, limit) +} + +func (m *Manager) GetErrorRow(ctx context.Context, id string) (ErrorRow, error) { + if err := m.ensureOpen(); err != nil { + return ErrorRow{}, err + } + return m.store.GetErrorRow(ctx, id) +} + +func (m *Manager) RetryErrorRow(ctx context.Context, id string, replay func(context.Context, ErrorRow) error) (ErrorRow, error) { + if err := m.ensureOpen(); err != nil { + return ErrorRow{}, err + } + if replay == nil { + return ErrorRow{}, errors.New("data sync error row retry callback is required") + } + claimed, err := m.store.ClaimErrorRowRetry(ctx, strings.TrimSpace(id), m.nowMillis(), errorRowRetryLeaseTTL) + if err != nil { + return ErrorRow{}, err + } + + replayCtx, cancelReplay := context.WithCancelCause(ctx) + heartbeatDone := make(chan error, 1) + go func() { + heartbeatDone <- m.maintainErrorRowRetryLease(replayCtx, cancelReplay, claimed.ID, claimed.RetryOwner) + }() + replayErr := callErrorRowRetry(replayCtx, claimed, replay) + cancelReplay(context.Canceled) + heartbeatErr := <-heartbeatDone + if heartbeatErr != nil { + replayErr = errors.Join(replayErr, heartbeatErr) + } + + finalizeCtx, cancelFinalize := context.WithTimeout(context.Background(), errorRowRetryFinalizeTTL) + defer cancelFinalize() + if replayErr != nil { + if err := m.store.FailErrorRowRetry(finalizeCtx, claimed.ID, claimed.RetryOwner, m.nowMillis()); err != nil { + return ErrorRow{}, errors.Join(replayErr, fmt.Errorf("release failed data sync error row retry: %w", err)) + } + row, readErr := m.store.GetErrorRow(finalizeCtx, claimed.ID) + if readErr != nil { + return ErrorRow{}, errors.Join(replayErr, readErr) + } + return row, replayErr + } + if err := m.store.ResolveErrorRowRetry(finalizeCtx, claimed.ID, claimed.RetryOwner, m.nowMillis()); err != nil { + return ErrorRow{}, err + } + return m.store.GetErrorRow(finalizeCtx, claimed.ID) +} + +func (m *Manager) maintainErrorRowRetryLease(ctx context.Context, cancel context.CancelCauseFunc, id, owner string) error { + ticker := time.NewTicker(errorRowRetryRenewInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-m.ctx.Done(): + cause := context.Cause(m.ctx) + if cause == nil { + cause = ErrManagerClosed + } + cancel(cause) + return cause + case <-ticker.C: + if err := m.store.RenewErrorRowRetry(context.Background(), id, owner, m.nowMillis(), errorRowRetryLeaseTTL); err != nil { + cancel(err) + return err + } + } + } +} + +func callErrorRowRetry(ctx context.Context, row ErrorRow, replay func(context.Context, ErrorRow) error) (err error) { + defer func() { + if recovered := recover(); recovered != nil { + err = fmt.Errorf("data sync error row retry panic: %v", recovered) + } + }() + return replay(ctx, row) +} + +func (m *Manager) GetCheckpoint(ctx context.Context, jobID string) (Checkpoint, error) { + if err := m.ensureOpen(); err != nil { + return Checkpoint{}, err + } + return m.store.GetCheckpoint(ctx, strings.TrimSpace(jobID)) +} + +func (m *Manager) ResetCheckpoint(ctx context.Context, jobID string) error { + if err := m.ensureOpen(); err != nil { + return err + } + return m.store.ResetCheckpoint(ctx, strings.TrimSpace(jobID)) +} + +func (m *Manager) ResolveErrorRow(ctx context.Context, id string, incrementAttempts bool) error { + if err := m.ensureOpen(); err != nil { + return err + } + return m.store.UpdateErrorRowStatus(ctx, id, ErrorRowResolved, incrementAttempts) +} + +func (m *Manager) RecordErrorRowRetryFailure(ctx context.Context, id string) error { + if err := m.ensureOpen(); err != nil { + return err + } + return m.store.IncrementErrorRowAttempts(ctx, strings.TrimSpace(id)) +} + +func (m *Manager) DiscardErrorRow(ctx context.Context, id string) error { + if err := m.ensureOpen(); err != nil { + return err + } + return m.store.UpdateErrorRowStatus(ctx, id, ErrorRowDiscarded, false) +} + +func (m *Manager) StartRun(ctx context.Context, jobID string) (RunRecord, error) { + if err := m.ensureOpen(); err != nil { + return RunRecord{}, err + } + definition, err := m.store.GetJob(ctx, strings.TrimSpace(jobID)) + if err != nil { + return RunRecord{}, err + } + if definition.Lifecycle != JobLifecycleReady && definition.Lifecycle != JobLifecycleEnabled { + return RunRecord{}, ErrJobDisabled + } + if err := ValidateDefinition(definition); err != nil { + return RunRecord{}, err + } + run, err := m.createRun(ctx, definition, RunTriggerManual, "", 1) + if err != nil { + return RunRecord{}, err + } + m.signalWake() + return run, nil +} + +func (m *Manager) CancelRun(ctx context.Context, runID string) error { + if err := m.ensureOpen(); err != nil { + return err + } + run, err := m.store.RequestCancelRun(ctx, strings.TrimSpace(runID), m.nowMillis()) + if err != nil { + return err + } + eventType := RunEventCancelling + message := "cancellation requested" + if run.Status == RunStatusCanceled { + eventType = RunEventCanceled + message = "canceled before execution" + } + _, eventErr := m.appendEvent(ctx, run, eventType, message, nil) + if run.Status == RunStatusCancelling { + m.cancelLocalRun(run.ID, errRunCanceled) + } + return eventErr +} + +func (m *Manager) ResumeRun(ctx context.Context, runID string) (RunRecord, error) { + if err := m.ensureOpen(); err != nil { + return RunRecord{}, err + } + parent, err := m.store.GetRun(ctx, strings.TrimSpace(runID)) + if err != nil { + return RunRecord{}, err + } + switch parent.Status { + case RunStatusFailed, RunStatusCanceled, RunStatusInterrupted, RunStatusPartial, RunStatusPaused: + default: + return RunRecord{}, ErrRunNotResumable + } + if !parent.Resumable { + return RunRecord{}, ErrRunNotResumable + } + definition, err := decodeRunDefinition(parent) + if err != nil { + return RunRecord{}, err + } + current, err := m.requireCurrentRunnableDefinition(ctx, parent, definition) + if err != nil { + return RunRecord{}, err + } + if current.ResumePolicy == "never" { + return RunRecord{}, ErrRunNotResumable + } + if current.Options.SyncMode == "insert_only" { + return RunRecord{}, ErrRunNotResumable + } + checkpoint, err := m.store.GetCheckpoint(ctx, parent.JobID) + if err != nil { + if errors.Is(err, ErrNotFound) { + return RunRecord{}, ErrRunNotResumable + } + return RunRecord{}, err + } + inLineage, err := m.checkpointBelongsToRunLineage(ctx, parent, checkpoint, current) + if err != nil { + return RunRecord{}, err + } + if !inLineage { + return RunRecord{}, ErrRunNotResumable + } + resumed, err := m.createRun(ctx, current, RunTriggerResume, parent.ID, parent.Attempt+1) + if err != nil { + return RunRecord{}, err + } + m.signalWake() + return resumed, nil +} + +func (m *Manager) RetryRun(ctx context.Context, runID string) (RunRecord, error) { + if err := m.ensureOpen(); err != nil { + return RunRecord{}, err + } + parent, err := m.store.GetRun(ctx, strings.TrimSpace(runID)) + if err != nil { + return RunRecord{}, err + } + switch parent.Status { + case RunStatusFailed, RunStatusPartial, RunStatusCanceled, RunStatusInterrupted: + default: + return RunRecord{}, ErrRunNotRetryable + } + definition, err := decodeRunDefinition(parent) + if err != nil { + return RunRecord{}, err + } + current, err := m.requireCurrentRunnableDefinition(ctx, parent, definition) + if err != nil { + return RunRecord{}, err + } + if current.Options.SyncMode == "insert_only" { + return RunRecord{}, ErrRunNotRetryable + } + retried, err := m.createRun(ctx, current, RunTriggerRetry, parent.ID, parent.Attempt+1) + if err != nil { + return RunRecord{}, err + } + m.signalWake() + return retried, nil +} + +func (m *Manager) checkpointBelongsToRunLineage(ctx context.Context, parent RunRecord, checkpoint Checkpoint, definition JobDefinition) (bool, error) { + if checkpoint.JobID != parent.JobID || strings.TrimSpace(checkpoint.RunID) == "" { + return false, nil + } + expectedPlanHash, err := ExecutionPlanHash(definition) + if err != nil { + return false, err + } + visited := make(map[string]struct{}) + candidate := parent + for { + if candidate.JobID != parent.JobID { + return false, nil + } + if _, repeated := visited[candidate.ID]; repeated { + return false, nil + } + visited[candidate.ID] = struct{}{} + if candidate.ID == checkpoint.RunID { + if checkpoint.DefinitionRevision != 0 && checkpoint.DefinitionRevision != candidate.JobRevision { + return false, nil + } + candidateDefinition, err := decodeRunDefinition(candidate) + if err != nil { + return false, err + } + candidatePlanHash, err := ExecutionPlanHash(candidateDefinition) + if err != nil { + return false, err + } + return candidatePlanHash == expectedPlanHash, nil + } + if strings.TrimSpace(candidate.ParentRunID) == "" { + return false, nil + } + candidate, err = m.store.GetRun(ctx, candidate.ParentRunID) + if err != nil { + return false, err + } + } +} + +func (m *Manager) requireCurrentRunnableDefinition(ctx context.Context, run RunRecord, snapshot JobDefinition) (JobDefinition, error) { + current, err := m.store.GetJob(ctx, run.JobID) + if err != nil { + return JobDefinition{}, err + } + if current.Lifecycle != JobLifecycleReady && current.Lifecycle != JobLifecycleEnabled { + return JobDefinition{}, ErrJobDisabled + } + snapshotHash, err := ExecutionPlanHash(snapshot) + if err != nil { + return JobDefinition{}, err + } + currentHash, err := ExecutionPlanHash(current) + if err != nil { + return JobDefinition{}, err + } + if snapshot.ID != run.JobID || snapshot.Revision != run.JobRevision || snapshotHash != currentHash { + return JobDefinition{}, fmt.Errorf("%w: the task execution plan changed after run %s", ErrRevisionConflict, run.ID) + } + return current, nil +} + +func (m *Manager) createRun(ctx context.Context, definition JobDefinition, trigger RunTrigger, parentRunID string, attempt int) (RunRecord, error) { + return m.createRunWithID(ctx, definition, trigger, parentRunID, attempt, "") +} + +func (m *Manager) createRunWithID(ctx context.Context, definition JobDefinition, trigger RunTrigger, parentRunID string, attempt int, runID string) (RunRecord, error) { + snapshot, err := json.Marshal(definition) + if err != nil { + return RunRecord{}, fmt.Errorf("encode data sync job run snapshot: %w", err) + } + run, event, err := m.store.CreateRunWithPolicyAndQueuedEvent(ctx, RunRecord{ + ID: runID, + JobID: definition.ID, + JobRevision: definition.Revision, + Trigger: trigger, + Status: RunStatusQueued, + ParentRunID: parentRunID, + Attempt: attempt, + DefinitionSnapshot: snapshot, + SourceFingerprint: definition.Source.Fingerprint, + TargetFingerprint: definition.Target.Fingerprint, + }, definition.ConcurrencyPolicy, m.nowMillis()) + if err != nil { + return RunRecord{}, err + } + m.notifyRunEvent(event) + return run, nil +} + +func (m *Manager) dispatchLoop() { + defer m.wg.Done() + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-m.ctx.Done(): + return + case <-m.wake: + case <-ticker.C: + } + m.dispatchQueued() + } +} + +func (m *Manager) schedulerLoop() { + defer m.wg.Done() + ticker := time.NewTicker(m.options.SchedulerInterval) + defer ticker.Stop() + for { + m.runSchedulerCycle() + select { + case <-m.ctx.Done(): + return + case <-ticker.C: + } + } +} + +func (m *Manager) runSchedulerCycle() { + if m.ctx.Err() != nil { + return + } + now := m.options.Now() + acquired, err := m.store.AcquireSchedulerLease(m.ctx, "data-sync-scheduler", m.options.LeaseOwner, now, m.options.LeaseTTL) + if err != nil || !acquired { + return + } + if m.lastRecoveryAt.IsZero() || now.Sub(m.lastRecoveryAt) >= m.options.RecoveryInterval { + if err := m.recoverInterrupted(m.ctx); err != nil { + return + } + m.lastRecoveryAt = now + } + dueJobs, err := m.store.ListDueJobs(m.ctx, now.UnixMilli()) + if err != nil { + return + } + for _, definition := range dueJobs { + if m.ctx.Err() != nil { + return + } + if definition.Schedule.Kind == ScheduleContinuous { + notBefore, _, err := m.continuousFailureNotBefore(m.ctx, definition) + if err != nil { + continue + } + if notBefore > now.UnixMilli() { + _, _ = m.store.DelayScheduleIfDue(m.ctx, definition.ID, definition.NextRunAt, notBefore) + continue + } + } + m.enqueueScheduled(definition, now) + } +} + +func (m *Manager) enqueueScheduled(definition JobDefinition, now time.Time) { + scheduledAt := definition.NextRunAt + if scheduledAt <= 0 { + return + } + runID := scheduledRunID(definition.ID, scheduledAt) + run, err := m.createRunWithID(m.ctx, definition, RunTriggerSchedule, "", 1, runID) + if err != nil { + if errors.Is(err, ErrRunAlreadyActive) { + _, _ = m.store.AdvanceScheduleIfDue(m.ctx, definition.ID, scheduledAt, now) + return + } + existing, getErr := m.store.GetRun(m.ctx, runID) + if getErr != nil { + return + } + run = existing + } + advanced, err := m.store.AdvanceScheduleIfDue(m.ctx, definition.ID, scheduledAt, now) + if err != nil { + return + } + if advanced || run.Status == RunStatusQueued { + m.signalWake() + } +} + +func scheduledRunID(jobID string, scheduledAt int64) string { + value := fmt.Sprintf("%s\x00%d", jobID, scheduledAt) + return "sync-run-scheduled-" + uuid.NewSHA1(uuid.NameSpaceOID, []byte(value)).String() +} + +func (m *Manager) dispatchQueued() { + if !m.hasDispatchCapacity() { + return + } + runs, err := m.store.ListQueuedRuns(m.ctx, 200) + if err != nil { + return + } + for _, queued := range runs { + if m.ctx.Err() != nil { + return + } + if !m.hasDispatchCapacity() { + return + } + run, claimed, err := m.store.ClaimRun(m.ctx, queued.ID, m.nowMillis()) + if err != nil || !claimed { + continue + } + if !m.launch(run) { + _, _ = m.store.CompleteRunOwned(context.Background(), run.ID, run.OwnerToken, RunStatusInterrupted, ExecutionOutcome{Resumable: true}, "manager stopped before execution", m.nowMillis()) + return + } + } +} + +func (m *Manager) hasDispatchCapacity() bool { + m.mu.Lock() + defer m.mu.Unlock() + return !m.closing && len(m.active) < m.options.MaxConcurrentRuns +} + +func (m *Manager) launch(run RunRecord) bool { + runCtx, cancel := context.WithCancelCause(m.ctx) + m.mu.Lock() + if m.closing || len(m.active) >= m.options.MaxConcurrentRuns { + m.mu.Unlock() + cancel(errManagerShutdown) + return false + } + m.active[run.ID] = activeExecution{jobID: run.JobID, ownerToken: run.OwnerToken, cancel: cancel} + m.wg.Add(1) + m.mu.Unlock() + current, err := m.store.GetRun(context.Background(), run.ID) + publishStarted := true + switch { + case err != nil: + cancel(err) + publishStarted = false + case current.OwnerToken != run.OwnerToken: + cancel(ErrRunOwnershipLost) + publishStarted = false + case current.Status == RunStatusCancelling: + cancel(errRunCanceled) + publishStarted = false + case current.Status != RunStatusRunning: + cancel(ErrRunOwnershipLost) + publishStarted = false + } + if publishStarted { + if _, err := m.appendEvent(context.Background(), run, RunEventStarted, "started", nil); err != nil { + cancel(err) + } + } + go m.execute(runCtx, run) + return true +} + +func (m *Manager) execute(ctx context.Context, run RunRecord) { + defer m.wg.Done() + defer func() { + m.mu.Lock() + delete(m.active, run.ID) + m.mu.Unlock() + m.signalWake() + }() + + definition, err := decodeRunDefinition(run) + if err != nil { + m.finish(run, RunStatusFailed, ExecutionOutcome{}, err.Error()) + return + } + if m.finishBeforeExecutionForCause(run, context.Cause(ctx)) { + return + } + var checkpoint *Checkpoint + if persisted, checkpointErr := m.store.GetCheckpoint(ctx, run.JobID); checkpointErr == nil { + checkpoint = &persisted + } else if !errors.Is(checkpointErr, ErrNotFound) { + if m.finishBeforeExecutionForCause(run, context.Cause(ctx)) { + return + } + m.finish(run, RunStatusFailed, ExecutionOutcome{}, checkpointErr.Error()) + return + } + if err := m.store.TouchRunOwned(ctx, run.ID, run.OwnerToken, m.nowMillis()); err != nil { + m.cancelLocalRun(run.ID, err) + if m.finishBeforeExecutionForCause(run, context.Cause(ctx)) { + return + } + m.finish(run, RunStatusFailed, ExecutionOutcome{}, err.Error()) + return + } + reporter := &managerReporter{manager: m, ctx: ctx, run: run} + stopHeartbeat := make(chan struct{}) + heartbeatDone := make(chan struct{}) + go func() { + defer close(heartbeatDone) + m.heartbeat(ctx, run.ID, run.OwnerToken, stopHeartbeat) + }() + var outcome ExecutionOutcome + var executeErr error + if cause := context.Cause(ctx); cause != nil { + executeErr = cause + } else { + outcome, executeErr = m.callExecutor(ctx, ExecutionRequest{Run: run, Definition: definition, Checkpoint: checkpoint}, reporter) + } + close(stopHeartbeat) + <-heartbeatDone + if executeErr == nil && context.Cause(ctx) == nil && outcome.RowsFailed == 0 && definition.IncrementalMode == IncrementalSnapshot { + if deleteErr := m.store.DeleteCheckpointOwned(context.Background(), run.JobID, run.ID, run.OwnerToken); deleteErr != nil { + executeErr = fmt.Errorf("clear completed snapshot checkpoint: %w", deleteErr) + outcome.Resumable = true + } + } + + status := RunStatusSucceeded + message := outcome.Message + if cause := context.Cause(ctx); cause != nil { + switch { + case errors.Is(cause, errManagerShutdown): + status = RunStatusInterrupted + outcome.Resumable = true + message = "manager stopped during execution" + case errors.Is(cause, errRunCanceled): + status = RunStatusCanceled + message = "canceled" + default: + status = RunStatusFailed + message = cause.Error() + } + } else if executeErr != nil { + status = RunStatusFailed + message = executeErr.Error() + } else if outcome.RowsFailed > 0 { + status = RunStatusPartial + } + completed := m.finish(run, status, outcome, message) + var permanentFailure *PermanentExecutionError + if completed && status == RunStatusFailed && executeErr != nil && errors.As(executeErr, &permanentFailure) { + _, _ = m.PauseJob(context.Background(), run.JobID) + } +} + +func (m *Manager) finishBeforeExecutionForCause(run RunRecord, cause error) bool { + if cause == nil { + return false + } + status := RunStatusFailed + message := cause.Error() + outcome := ExecutionOutcome{} + switch { + case errors.Is(cause, errManagerShutdown): + status = RunStatusInterrupted + message = "manager stopped before execution" + outcome.Resumable = true + case errors.Is(cause, errRunCanceled): + status = RunStatusCanceled + message = "canceled" + } + m.finish(run, status, outcome, message) + return true +} + +func (m *Manager) callExecutor(ctx context.Context, request ExecutionRequest, reporter RunReporter) (outcome ExecutionOutcome, err error) { + defer func() { + if recovered := recover(); recovered != nil { + err = fmt.Errorf("data sync executor panic: %v", recovered) + } + }() + return m.executor.Execute(ctx, request, reporter) +} + +func (m *Manager) finish(run RunRecord, status RunStatus, outcome ExecutionOutcome, message string) bool { + completed, err := m.store.CompleteRunOwned(context.Background(), run.ID, run.OwnerToken, status, outcome, message, m.nowMillis()) + if err != nil { + return false + } + _, _ = m.appendEvent(context.Background(), completed, eventTypeForStatus(status), message, nil) + return true +} + +func (m *Manager) heartbeat(ctx context.Context, runID, ownerToken string, stop <-chan struct{}) { + ticker := time.NewTicker(m.options.HeartbeatInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-stop: + return + case <-ticker.C: + run, err := m.store.GetRun(context.Background(), runID) + if err != nil { + m.cancelLocalRun(runID, err) + return + } + if run.OwnerToken != ownerToken { + m.cancelLocalRun(runID, ErrRunOwnershipLost) + return + } + if run.Status == RunStatusCancelling { + m.cancelLocalRun(runID, errRunCanceled) + return + } + if err := m.store.TouchRunOwned(context.Background(), runID, ownerToken, m.nowMillis()); err != nil { + m.cancelLocalRun(runID, err) + return + } + } + } +} + +func (m *Manager) appendEvent(ctx context.Context, run RunRecord, eventType RunEventType, message string, payload json.RawMessage) (RunEvent, error) { + event, err := m.store.AppendRunEvent(ctx, RunEvent{ + RunID: run.ID, + JobID: run.JobID, + Type: eventType, + Status: run.Status, + Current: run.Current, + Total: run.Total, + Table: run.Table, + Stage: run.Stage, + Message: message, + Payload: payload, + CreatedAt: m.nowMillis(), + }) + if err != nil { + return RunEvent{}, err + } + m.notifyRunEvent(event) + return event, nil +} + +func (m *Manager) notifyRunEvent(event RunEvent) { + hook := m.options.Hooks.OnRunEvent + if hook == nil { + return + } + defer func() { _ = recover() }() + hook(event) +} + +func (m *Manager) signalWake() { + select { + case m.wake <- struct{}{}: + default: + } +} + +func (m *Manager) cancelLocalRun(runID string, cause error) { + m.mu.Lock() + execution, ok := m.active[runID] + m.mu.Unlock() + if ok && execution.cancel != nil { + execution.cancel(cause) + } +} + +func (m *Manager) cancelLocalJobRuns(jobID string, cause error) { + m.mu.Lock() + cancellations := make([]context.CancelCauseFunc, 0) + for _, execution := range m.active { + if execution.jobID == jobID && execution.cancel != nil { + cancellations = append(cancellations, execution.cancel) + } + } + m.mu.Unlock() + for _, cancel := range cancellations { + cancel(cause) + } +} + +func (m *Manager) ensureOpen() error { + if m == nil { + return ErrManagerClosed + } + m.mu.Lock() + closing := m.closing + m.mu.Unlock() + if closing { + return ErrManagerClosed + } + return nil +} + +func (m *Manager) nowMillis() int64 { + return m.options.Now().UnixMilli() +} + +func (m *Manager) Shutdown(ctx context.Context) error { + if m == nil { + return nil + } + if ctx == nil { + ctx = context.Background() + } + m.shutdownOnce.Do(func() { + m.mu.Lock() + m.closing = true + m.cancel(errManagerShutdown) + m.mu.Unlock() + _ = m.store.ReleaseSchedulerLease(context.Background(), "data-sync-scheduler", m.options.LeaseOwner) + go func() { + m.wg.Wait() + close(m.done) + }() + }) + select { + case <-m.done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func decodeRunDefinition(run RunRecord) (JobDefinition, error) { + var definition JobDefinition + if err := json.Unmarshal(run.DefinitionSnapshot, &definition); err != nil { + return JobDefinition{}, fmt.Errorf("decode data sync job run snapshot: %w", err) + } + definition = NormalizeDefinition(definition) + if err := ValidateDefinition(definition); err != nil { + return JobDefinition{}, fmt.Errorf("validate data sync job run snapshot: %w", err) + } + if definition.ID != run.JobID || definition.Revision != run.JobRevision { + return JobDefinition{}, fmt.Errorf("data sync job run snapshot identity does not match run %s", run.ID) + } + if definition.Source.Fingerprint != run.SourceFingerprint || definition.Target.Fingerprint != run.TargetFingerprint { + return JobDefinition{}, fmt.Errorf("data sync job run snapshot endpoint fingerprints do not match run %s", run.ID) + } + return definition, nil +} + +func eventTypeForStatus(status RunStatus) RunEventType { + switch status { + case RunStatusSucceeded: + return RunEventSucceeded + case RunStatusPartial: + return RunEventPartial + case RunStatusCanceled: + return RunEventCanceled + case RunStatusInterrupted: + return RunEventInterrupted + default: + return RunEventFailed + } +} + +type managerReporter struct { + manager *Manager + ctx context.Context + run RunRecord + mu sync.Mutex +} + +func (r *managerReporter) ReportProgress(progress RunProgress) error { + r.mu.Lock() + defer r.mu.Unlock() + run, err := r.manager.store.UpdateRunProgressOwned(r.ctx, r.run.ID, r.run.OwnerToken, progress, r.manager.nowMillis()) + if err != nil { + return err + } + r.run = run + payload, _ := json.Marshal(progress) + _, err = r.manager.appendEvent(r.ctx, run, RunEventProgress, progress.Message, payload) + return err +} + +func (r *managerReporter) SaveCheckpoint(checkpoint Checkpoint) error { + r.mu.Lock() + defer r.mu.Unlock() + checkpoint.JobID = r.run.JobID + checkpoint.RunID = r.run.ID + checkpoint.DefinitionRevision = r.run.JobRevision + persisted, err := r.manager.store.PutCheckpointOwned(r.ctx, checkpoint, r.run.OwnerToken) + if err != nil { + return err + } + publicCheckpoint := persisted + publicCheckpoint.SchemaHash = "" + payload, _ := json.Marshal(publicCheckpoint) + _, err = r.manager.appendEvent(r.ctx, r.run, RunEventCheckpoint, "checkpoint saved", payload) + return err +} + +func (r *managerReporter) AppendErrorRow(row ErrorRow) error { + r.mu.Lock() + defer r.mu.Unlock() + row.RunID = r.run.ID + row.JobID = r.run.JobID + persisted, err := r.manager.store.AppendErrorRow(r.ctx, row) + if err != nil { + return err + } + // Error events are notification metadata only. Source keys and captured row + // payloads stay in the explicit error-row store and are never broadcast. + payload, _ := json.Marshal(struct { + ID string `json:"id"` + SourceTable string `json:"sourceTable,omitempty"` + TargetTable string `json:"targetTable,omitempty"` + Operation string `json:"operation,omitempty"` + PayloadPolicy string `json:"payloadPolicy,omitempty"` + PayloadHash string `json:"payloadHash,omitempty"` + PayloadSize int64 `json:"payloadSize,omitempty"` + ErrorCode string `json:"errorCode,omitempty"` + ErrorClass string `json:"errorClass,omitempty"` + Status ErrorRowStatus `json:"status"` + }{ + ID: persisted.ID, + SourceTable: persisted.SourceTable, + TargetTable: persisted.TargetTable, + Operation: persisted.Operation, + PayloadPolicy: persisted.PayloadPolicy, + PayloadHash: persisted.PayloadHash, + PayloadSize: persisted.PayloadSize, + ErrorCode: persisted.ErrorCode, + ErrorClass: persisted.ErrorClass, + Status: persisted.Status, + }) + _, err = r.manager.appendEvent(r.ctx, r.run, RunEventErrorRow, persisted.Error, payload) + return err +} + +func (r *managerReporter) Emit(eventType RunEventType, message string, payload json.RawMessage) error { + r.mu.Lock() + defer r.mu.Unlock() + if eventType == "" { + eventType = RunEventLog + } + _, err := r.manager.appendEvent(r.ctx, r.run, eventType, message, payload) + return err +} diff --git a/internal/syncjob/manager_test.go b/internal/syncjob/manager_test.go new file mode 100644 index 00000000..31baf65d --- /dev/null +++ b/internal/syncjob/manager_test.go @@ -0,0 +1,883 @@ +package syncjob + +import ( + "context" + "encoding/json" + "errors" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestManagerQueuesRunsForTheSameJobWithoutOverlap(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "queue") + + started := make(chan string, 2) + release := make(chan struct{}, 2) + var active atomic.Int32 + var maxActive atomic.Int32 + executor := ExecutorFunc(func(ctx context.Context, request ExecutionRequest, _ RunReporter) (ExecutionOutcome, error) { + current := active.Add(1) + defer active.Add(-1) + for { + maximum := maxActive.Load() + if current <= maximum || maxActive.CompareAndSwap(maximum, current) { + break + } + } + started <- request.Run.ID + select { + case <-ctx.Done(): + return ExecutionOutcome{}, context.Cause(ctx) + case <-release: + return ExecutionOutcome{RowsInserted: 1}, nil + } + }) + manager, err := NewManager(context.Background(), store, executor, ManagerOptions{ + SchedulerInterval: time.Hour, + HeartbeatInterval: time.Hour, + }) + if err != nil { + t.Fatalf("new manager: %v", err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := manager.Shutdown(ctx); err != nil { + t.Errorf("shutdown manager: %v", err) + } + }) + + first, err := manager.StartRun(context.Background(), definition.ID) + if err != nil { + t.Fatalf("start first run: %v", err) + } + if got := receiveString(t, started); got != first.ID { + t.Fatalf("first executed run = %q, want %q", got, first.ID) + } + second, err := manager.StartRun(context.Background(), definition.ID) + if err != nil { + t.Fatalf("queue second run: %v", err) + } + assertRunStatus(t, store, second.ID, RunStatusQueued) + select { + case got := <-started: + t.Fatalf("second run %q overlapped first run", got) + case <-time.After(75 * time.Millisecond): + } + + release <- struct{}{} + waitRunStatus(t, store, first.ID, RunStatusSucceeded) + if got := receiveString(t, started); got != second.ID { + t.Fatalf("second executed run = %q, want %q", got, second.ID) + } + release <- struct{}{} + waitRunStatus(t, store, second.ID, RunStatusSucceeded) + if got := maxActive.Load(); got != 1 { + t.Fatalf("maximum concurrent executions = %d, want 1", got) + } + + for _, runID := range []string{first.ID, second.ID} { + events, err := store.ListRunEvents(context.Background(), runID, 0, 20) + if err != nil { + t.Fatalf("list events for %s: %v", runID, err) + } + if len(events) != 3 { + t.Fatalf("event count for %s = %d, want 3: %#v", runID, len(events), events) + } + for index, event := range events { + if event.Sequence != int64(index+1) { + t.Fatalf("event sequence at %d = %d, want %d", index, event.Sequence, index+1) + } + } + } +} + +func TestManagerCancelsQueuedAndRunningRuns(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "queue") + started := make(chan string, 2) + exited := make(chan struct{}, 1) + executor := ExecutorFunc(func(ctx context.Context, request ExecutionRequest, _ RunReporter) (ExecutionOutcome, error) { + started <- request.Run.ID + <-ctx.Done() + exited <- struct{}{} + return ExecutionOutcome{}, context.Cause(ctx) + }) + manager := newTestManager(t, store, executor) + + first, err := manager.StartRun(context.Background(), definition.ID) + if err != nil { + t.Fatalf("start first run: %v", err) + } + if got := receiveString(t, started); got != first.ID { + t.Fatalf("executed run = %q, want %q", got, first.ID) + } + second, err := manager.StartRun(context.Background(), definition.ID) + if err != nil { + t.Fatalf("queue second run: %v", err) + } + if err := manager.CancelRun(context.Background(), second.ID); err != nil { + t.Fatalf("cancel queued run: %v", err) + } + waitRunStatus(t, store, second.ID, RunStatusCanceled) + if err := manager.CancelRun(context.Background(), first.ID); err != nil { + t.Fatalf("cancel running run: %v", err) + } + waitRunStatus(t, store, first.ID, RunStatusCanceled) + select { + case <-exited: + case <-time.After(time.Second): + t.Fatal("executor did not observe cancellation") + } + select { + case got := <-started: + t.Fatalf("canceled queued run unexpectedly executed: %s", got) + case <-time.After(75 * time.Millisecond): + } + events, err := store.ListRunEvents(context.Background(), first.ID, 0, 20) + if err != nil { + t.Fatalf("list canceled run events: %v", err) + } + if len(events) < 4 || events[len(events)-2].Type != RunEventCancelling || events[len(events)-1].Type != RunEventCanceled { + t.Fatalf("cancellation event order = %#v", events) + } +} + +func TestManagerForbidPolicyRejectsAnUnfinishedRun(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "forbid") + started := make(chan string, 1) + executor := ExecutorFunc(func(ctx context.Context, request ExecutionRequest, _ RunReporter) (ExecutionOutcome, error) { + started <- request.Run.ID + <-ctx.Done() + return ExecutionOutcome{}, context.Cause(ctx) + }) + manager := newTestManager(t, store, executor) + first, err := manager.StartRun(context.Background(), definition.ID) + if err != nil { + t.Fatalf("start first run: %v", err) + } + if got := receiveString(t, started); got != first.ID { + t.Fatalf("executed run = %q, want %q", got, first.ID) + } + if _, err := manager.StartRun(context.Background(), definition.ID); !errors.Is(err, ErrRunAlreadyActive) { + t.Fatalf("start overlapping run error = %v, want ErrRunAlreadyActive", err) + } +} + +func TestManagerResumesAFailedRunFromItsCheckpoint(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "forbid") + requests := make(chan ExecutionRequest, 2) + var calls atomic.Int32 + executor := ExecutorFunc(func(_ context.Context, request ExecutionRequest, reporter RunReporter) (ExecutionOutcome, error) { + requests <- request + if calls.Add(1) == 1 { + if err := reporter.SaveCheckpoint(Checkpoint{ + Kind: "watermark", + Table: "orders", + Phase: "copy", + CursorType: "primary_key", + Cursor: []byte(`{"id":42}`), + }); err != nil { + return ExecutionOutcome{}, err + } + return ExecutionOutcome{RowsInserted: 42, Resumable: true}, errors.New("target unavailable") + } + return ExecutionOutcome{RowsInserted: 1}, nil + }) + manager := newTestManager(t, store, executor) + + failed, err := manager.StartRun(context.Background(), definition.ID) + if err != nil { + t.Fatalf("start run: %v", err) + } + waitRunStatus(t, store, failed.ID, RunStatusFailed) + resumed, err := manager.ResumeRun(context.Background(), failed.ID) + if err != nil { + t.Fatalf("resume run: %v", err) + } + if resumed.ParentRunID != failed.ID || resumed.Attempt != 2 || resumed.Trigger != RunTriggerResume { + t.Fatalf("resumed run lineage = %#v", resumed) + } + waitRunStatus(t, store, resumed.ID, RunStatusSucceeded) + + firstRequest := receiveRequest(t, requests) + secondRequest := receiveRequest(t, requests) + if firstRequest.Checkpoint != nil { + t.Fatalf("initial request unexpectedly received checkpoint: %#v", firstRequest.Checkpoint) + } + if secondRequest.Checkpoint == nil || secondRequest.Checkpoint.RunID != failed.ID || string(secondRequest.Checkpoint.Cursor) != `{"id":42}` { + t.Fatalf("resume checkpoint = %#v", secondRequest.Checkpoint) + } +} + +func TestManagerClearsCheckpointAfterSuccessfulSnapshot(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "forbid") + executor := ExecutorFunc(func(_ context.Context, _ ExecutionRequest, reporter RunReporter) (ExecutionOutcome, error) { + if err := reporter.SaveCheckpoint(Checkpoint{ + Version: 1, + Kind: "resume", + Table: "orders", + Phase: "mapping_completed", + CursorType: "mapping_index", + Cursor: json.RawMessage(`{"nextMapping":1}`), + }); err != nil { + return ExecutionOutcome{}, err + } + return ExecutionOutcome{RowsInserted: 1}, nil + }) + manager, err := NewManager(context.Background(), store, executor, ManagerOptions{ + SchedulerInterval: time.Hour, + HeartbeatInterval: time.Hour, + }) + if err != nil { + t.Fatalf("new manager: %v", err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = manager.Shutdown(ctx) + }) + run, err := manager.StartRun(context.Background(), definition.ID) + if err != nil { + t.Fatalf("start run: %v", err) + } + waitRunStatus(t, store, run.ID, RunStatusSucceeded) + if _, err := store.GetCheckpoint(context.Background(), definition.ID); !errors.Is(err, ErrNotFound) { + t.Fatalf("completed snapshot checkpoint error = %v, want ErrNotFound", err) + } +} + +func TestManagerKeepsCheckpointAfterSuccessfulWatermarkRun(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "forbid") + definition.IncrementalMode = IncrementalWatermark + definition.Mappings[0].Watermark = &WatermarkSpec{Column: "updated_at", TieBreakerColumns: []string{"id"}} + definition, err := store.PutJob(context.Background(), definition) + if err != nil { + t.Fatalf("update watermark definition: %v", err) + } + executor := ExecutorFunc(func(_ context.Context, request ExecutionRequest, reporter RunReporter) (ExecutionOutcome, error) { + if request.Definition.IncrementalMode != IncrementalWatermark { + return ExecutionOutcome{}, errors.New("executor received non-watermark definition") + } + if err := reporter.SaveCheckpoint(Checkpoint{ + Version: 1, + Kind: "watermark", + Table: "orders", + Phase: "batch_committed", + CursorType: "watermark_map", + Cursor: json.RawMessage(`{"orders":{"updatedAt":"2026-08-08T00:00:00Z","id":42}}`), + }); err != nil { + return ExecutionOutcome{}, err + } + return ExecutionOutcome{RowsUpdated: 1}, nil + }) + manager := newTestManager(t, store, executor) + run, err := manager.StartRun(context.Background(), definition.ID) + if err != nil { + t.Fatalf("start run: %v", err) + } + waitRunStatus(t, store, run.ID, RunStatusSucceeded) + checkpoint, err := store.GetCheckpoint(context.Background(), definition.ID) + if err != nil { + t.Fatalf("get watermark checkpoint: %v", err) + } + if checkpoint.RunID != run.ID || checkpoint.Kind != "watermark" { + t.Fatalf("unexpected watermark checkpoint: %#v", checkpoint) + } +} + +func TestManagerRecoversStaleRunningAndQueuedRunsOnStartup(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "queue") + snapshot, err := json.Marshal(definition) + if err != nil { + t.Fatalf("marshal definition: %v", err) + } + stale, err := store.CreateRun(context.Background(), RunRecord{ + JobID: definition.ID, + JobRevision: definition.Revision, + Status: RunStatusRunning, + StartedAt: time.Now().Add(-time.Hour).UnixMilli(), + HeartbeatAt: time.Now().Add(-time.Hour).UnixMilli(), + DefinitionSnapshot: snapshot, + }) + if err != nil { + t.Fatalf("create stale run: %v", err) + } + queued, err := store.CreateRun(context.Background(), RunRecord{ + JobID: definition.ID, + JobRevision: definition.Revision, + Status: RunStatusQueued, + DefinitionSnapshot: snapshot, + }) + if err != nil { + t.Fatalf("create queued run: %v", err) + } + executed := make(chan string, 1) + manager, err := NewManager(context.Background(), store, ExecutorFunc(func(_ context.Context, request ExecutionRequest, _ RunReporter) (ExecutionOutcome, error) { + executed <- request.Run.ID + return ExecutionOutcome{}, nil + }), ManagerOptions{ + SchedulerInterval: time.Hour, + HeartbeatInterval: time.Hour, + RecoveryStaleAfter: time.Second, + }) + if err != nil { + t.Fatalf("new manager: %v", err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := manager.Shutdown(ctx); err != nil { + t.Errorf("shutdown manager: %v", err) + } + }) + + recovered := waitRunStatus(t, store, stale.ID, RunStatusInterrupted) + if !recovered.Resumable || recovered.FinishedAt == 0 { + t.Fatalf("recovered stale run = %#v", recovered) + } + if got := receiveString(t, executed); got != queued.ID { + t.Fatalf("restored queued run = %q, want %q", got, queued.ID) + } + waitRunStatus(t, store, queued.ID, RunStatusSucceeded) + events, err := store.ListRunEvents(context.Background(), stale.ID, 0, 10) + if err != nil { + t.Fatalf("list stale run events: %v", err) + } + if len(events) != 1 || events[0].Type != RunEventInterrupted { + t.Fatalf("stale run events = %#v", events) + } +} + +func TestManagersUseSQLiteLeaseToScheduleOneRun(t *testing.T) { + databasePath := t.TempDir() + "/shared-sync-jobs.db" + firstStore, err := Open(databasePath) + if err != nil { + t.Fatalf("open first store: %v", err) + } + t.Cleanup(func() { _ = firstStore.Close() }) + secondStore, err := Open(databasePath) + if err != nil { + t.Fatalf("open second store: %v", err) + } + t.Cleanup(func() { _ = secondStore.Close() }) + definition, err := firstStore.PutJob(context.Background(), JobDefinition{ + Name: "scheduled orders sync", + Enabled: true, + Kind: JobKindReconcile, + IncrementalMode: IncrementalSnapshot, + Source: EndpointRef{ConnectionID: "source"}, + Target: EndpointRef{ConnectionID: "target"}, + Mappings: []TableMapping{{SourceTable: "orders", TargetTable: "orders", Enabled: true}}, + ConcurrencyPolicy: "queue", + Schedule: ScheduleSpec{ + Kind: ScheduleInterval, + IntervalSeconds: 10, + MisfirePolicy: "run_once", + }, + }) + if err != nil { + t.Fatalf("put scheduled job: %v", err) + } + dueAt := time.Now().Add(-time.Second).UnixMilli() + if _, err := firstStore.db.ExecContext(context.Background(), `UPDATE data_sync_jobs SET next_run_at = ? WHERE id = ?`, dueAt, definition.ID); err != nil { + t.Fatalf("make job due: %v", err) + } + + var executions atomic.Int32 + executor := ExecutorFunc(func(context.Context, ExecutionRequest, RunReporter) (ExecutionOutcome, error) { + executions.Add(1) + return ExecutionOutcome{}, nil + }) + firstManager := newScheduledTestManager(t, firstStore, executor, "owner-a") + _ = firstManager + secondManager := newScheduledTestManager(t, secondStore, executor, "owner-b") + _ = secondManager + + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + runs, listErr := firstStore.ListRuns(context.Background(), definition.ID, 10) + if listErr == nil && len(runs) == 1 && runs[0].Status == RunStatusSucceeded { + if runs[0].Trigger != RunTriggerSchedule { + t.Fatalf("scheduled run trigger = %q", runs[0].Trigger) + } + break + } + time.Sleep(10 * time.Millisecond) + } + time.Sleep(100 * time.Millisecond) + runs, err := firstStore.ListRuns(context.Background(), definition.ID, 10) + if err != nil { + t.Fatalf("list scheduled runs: %v", err) + } + if len(runs) != 1 || executions.Load() != 1 { + t.Fatalf("scheduled runs = %d, executions = %d; want one", len(runs), executions.Load()) + } +} + +func TestManagerShutdownCancelsExecutorsAndWaitsForGoroutines(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "queue") + started := make(chan struct{}) + exited := make(chan struct{}) + executor := ExecutorFunc(func(ctx context.Context, _ ExecutionRequest, _ RunReporter) (ExecutionOutcome, error) { + close(started) + <-ctx.Done() + time.Sleep(25 * time.Millisecond) + close(exited) + return ExecutionOutcome{Resumable: true}, context.Cause(ctx) + }) + manager, err := NewManager(context.Background(), store, executor, ManagerOptions{ + SchedulerInterval: time.Hour, + HeartbeatInterval: time.Hour, + }) + if err != nil { + t.Fatalf("new manager: %v", err) + } + run, err := manager.StartRun(context.Background(), definition.ID) + if err != nil { + t.Fatalf("start run: %v", err) + } + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("executor did not start") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := manager.Shutdown(ctx); err != nil { + t.Fatalf("shutdown manager: %v", err) + } + select { + case <-exited: + default: + t.Fatal("shutdown returned before executor goroutine exited") + } + recovered := waitRunStatus(t, store, run.ID, RunStatusInterrupted) + if !recovered.Resumable { + t.Fatalf("shutdown-interrupted run is not resumable: %#v", recovered) + } +} + +func TestManagerAutomaticallyResumesRecoveredRunWhenConfigured(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "forbid") + definition.ResumePolicy = "auto" + definition, err := store.PutJob(context.Background(), definition) + if err != nil { + t.Fatalf("enable auto resume: %v", err) + } + snapshot, err := json.Marshal(definition) + if err != nil { + t.Fatalf("marshal definition: %v", err) + } + stale, err := store.CreateRun(context.Background(), RunRecord{ + JobID: definition.ID, + JobRevision: definition.Revision, + Status: RunStatusRunning, + StartedAt: time.Now().Add(-time.Hour).UnixMilli(), + HeartbeatAt: time.Now().Add(-time.Hour).UnixMilli(), + DefinitionSnapshot: snapshot, + }) + if err != nil { + t.Fatalf("create stale run: %v", err) + } + if _, err := store.PutCheckpoint(context.Background(), Checkpoint{ + Kind: "watermark", + JobID: definition.ID, + RunID: stale.ID, + DefinitionRevision: definition.Revision, + Table: "orders", + Phase: "copy", + CursorType: "primary_key", + Cursor: []byte(`{"id":99}`), + }); err != nil { + t.Fatalf("put checkpoint: %v", err) + } + requests := make(chan ExecutionRequest, 1) + manager, err := NewManager(context.Background(), store, ExecutorFunc(func(_ context.Context, request ExecutionRequest, _ RunReporter) (ExecutionOutcome, error) { + requests <- request + return ExecutionOutcome{}, nil + }), ManagerOptions{ + SchedulerInterval: time.Hour, + HeartbeatInterval: time.Hour, + RecoveryStaleAfter: time.Second, + }) + if err != nil { + t.Fatalf("new manager: %v", err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = manager.Shutdown(ctx) + }) + request := receiveRequest(t, requests) + if request.Run.Trigger != RunTriggerResume || request.Run.ParentRunID != stale.ID || request.Checkpoint == nil || request.Checkpoint.RunID != stale.ID { + t.Fatalf("automatic resume request = %#v", request) + } + waitRunStatus(t, store, request.Run.ID, RunStatusSucceeded) +} + +func TestManagerPersistsReporterOutputBeforePublishingHooks(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "queue") + var hookMu sync.Mutex + hooked := make([]RunEvent, 0) + executor := ExecutorFunc(func(_ context.Context, _ ExecutionRequest, reporter RunReporter) (ExecutionOutcome, error) { + if err := reporter.ReportProgress(RunProgress{Current: 3, Total: 5, Table: "orders", Stage: "write", Message: "batch 3"}); err != nil { + return ExecutionOutcome{}, err + } + if err := reporter.AppendErrorRow(ErrorRow{Error: "duplicate key", SourceTable: "orders", TargetTable: "orders", SourceKey: []byte(`{"id":3}`)}); err != nil { + return ExecutionOutcome{}, err + } + if err := reporter.Emit(RunEventLog, "executor log", []byte(`{"level":"info"}`)); err != nil { + return ExecutionOutcome{}, err + } + return ExecutionOutcome{RowsInserted: 2, RowsFailed: 1, Message: "completed with errors"}, nil + }) + manager, err := NewManager(context.Background(), store, executor, ManagerOptions{ + SchedulerInterval: time.Hour, + HeartbeatInterval: time.Hour, + Hooks: ManagerHooks{OnRunEvent: func(event RunEvent) { + persisted, listErr := store.ListRunEvents(context.Background(), event.RunID, event.Sequence-1, 1) + if listErr != nil || len(persisted) != 1 || persisted[0].Sequence != event.Sequence { + t.Errorf("hook observed event before persistence: event=%#v persisted=%#v err=%v", event, persisted, listErr) + } + hookMu.Lock() + hooked = append(hooked, event) + hookMu.Unlock() + }}, + }) + if err != nil { + t.Fatalf("new manager: %v", err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = manager.Shutdown(ctx) + }) + run, err := manager.StartRun(context.Background(), definition.ID) + if err != nil { + t.Fatalf("start run: %v", err) + } + completed := waitRunStatus(t, store, run.ID, RunStatusPartial) + if completed.Current != 3 || completed.Total != 5 || completed.RowsInserted != 2 || completed.RowsFailed != 1 { + t.Fatalf("persisted run output = %#v", completed) + } + errorRows, err := store.ListErrorRows(context.Background(), run.ID, ErrorRowPending, 10) + if err != nil || len(errorRows) != 1 || errorRows[0].Error != "duplicate key" { + t.Fatalf("persisted error rows = %#v, err=%v", errorRows, err) + } + wantTypes := []RunEventType{RunEventQueued, RunEventStarted, RunEventProgress, RunEventErrorRow, RunEventLog, RunEventPartial} + events := waitRunEventCount(t, store, run.ID, len(wantTypes)) + for index, want := range wantTypes { + if events[index].Type != want { + t.Fatalf("event %d type = %q, want %q", index, events[index].Type, want) + } + } + var hookCount int + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + hookMu.Lock() + hookCount = len(hooked) + hookMu.Unlock() + if hookCount == len(events) { + break + } + time.Sleep(time.Millisecond) + } + if hookCount != len(events) { + t.Fatalf("hook event count = %d, persisted = %d", hookCount, len(events)) + } +} + +func TestManagerRetriesTerminalRunWithOriginalSnapshotAndLineage(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "forbid") + requests := make(chan ExecutionRequest, 2) + var calls atomic.Int32 + executor := ExecutorFunc(func(_ context.Context, request ExecutionRequest, _ RunReporter) (ExecutionOutcome, error) { + requests <- request + if calls.Add(1) == 1 { + return ExecutionOutcome{}, errors.New("temporary target failure") + } + return ExecutionOutcome{}, nil + }) + manager := newTestManager(t, store, executor) + + failed, err := manager.StartRun(context.Background(), definition.ID) + if err != nil { + t.Fatalf("start run: %v", err) + } + waitRunStatus(t, store, failed.ID, RunStatusFailed) + retried, err := manager.RetryRun(context.Background(), failed.ID) + if err != nil { + t.Fatalf("retry failed run: %v", err) + } + if retried.Trigger != RunTriggerRetry || retried.ParentRunID != failed.ID || retried.Attempt != failed.Attempt+1 { + t.Fatalf("retry lineage = %#v", retried) + } + waitRunStatus(t, store, retried.ID, RunStatusSucceeded) + _ = receiveRequest(t, requests) + retryRequest := receiveRequest(t, requests) + if retryRequest.Definition.ID != definition.ID || retryRequest.Definition.Revision != definition.Revision { + t.Fatalf("retry definition snapshot = %#v", retryRequest.Definition) + } + if _, err := manager.RetryRun(context.Background(), retried.ID); !errors.Is(err, ErrRunNotRetryable) { + t.Fatalf("retry succeeded run error = %v, want ErrRunNotRetryable", err) + } +} + +func TestManagerRejectsRetryWhenOriginalSnapshotDoesNotMatchRun(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "forbid") + snapshot, err := json.Marshal(definition) + if err != nil { + t.Fatalf("marshal definition: %v", err) + } + run, err := store.CreateRun(context.Background(), RunRecord{ + JobID: definition.ID, + JobRevision: definition.Revision + 1, + Status: RunStatusFailed, + DefinitionSnapshot: snapshot, + }) + if err != nil { + t.Fatalf("create inconsistent run: %v", err) + } + manager := newTestManager(t, store, ExecutorFunc(func(context.Context, ExecutionRequest, RunReporter) (ExecutionOutcome, error) { + t.Fatal("inconsistent retry must not execute") + return ExecutionOutcome{}, nil + })) + if _, err := manager.RetryRun(context.Background(), run.ID); err == nil || errors.Is(err, ErrRunNotRetryable) { + t.Fatalf("inconsistent retry error = %v, want snapshot consistency error", err) + } +} + +func TestManagerRetriesAfterMetadataOnlyTaskRevisionChanges(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "forbid") + manager := newTestManager(t, store, ExecutorFunc(func(context.Context, ExecutionRequest, RunReporter) (ExecutionOutcome, error) { + return ExecutionOutcome{}, errors.New("temporary target failure") + })) + failed, err := manager.StartRun(context.Background(), definition.ID) + if err != nil { + t.Fatalf("start failing run: %v", err) + } + waitRunStatus(t, store, failed.ID, RunStatusFailed) + definition.Name = "renamed task" + if _, err := store.PutJob(context.Background(), definition); err != nil { + t.Fatalf("update task: %v", err) + } + updated, err := store.GetJob(context.Background(), definition.ID) + if err != nil { + t.Fatalf("get renamed task: %v", err) + } + retried, err := manager.RetryRun(context.Background(), failed.ID) + if err != nil { + t.Fatalf("retry after metadata change: %v", err) + } + if retried.JobRevision != updated.Revision { + t.Fatalf("retry revision = %d, want current %d", retried.JobRevision, updated.Revision) + } +} + +func TestManagerRejectsRetryAfterExecutionPlanChanges(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "forbid") + manager := newTestManager(t, store, ExecutorFunc(func(context.Context, ExecutionRequest, RunReporter) (ExecutionOutcome, error) { + return ExecutionOutcome{}, errors.New("temporary target failure") + })) + failed, err := manager.StartRun(context.Background(), definition.ID) + if err != nil { + t.Fatalf("start failing run: %v", err) + } + waitRunStatus(t, store, failed.ID, RunStatusFailed) + definition.Mappings[0].TargetTable = "orders_v2" + if _, err := store.PutJob(context.Background(), definition); err != nil { + t.Fatalf("update task plan: %v", err) + } + if _, err := manager.RetryRun(context.Background(), failed.ID); !errors.Is(err, ErrRevisionConflict) { + t.Fatalf("retry changed plan error = %v, want ErrRevisionConflict", err) + } +} + +func TestManagerRetryObeysForbidConcurrencyPolicy(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "forbid") + started := make(chan string, 1) + var calls atomic.Int32 + executor := ExecutorFunc(func(ctx context.Context, request ExecutionRequest, _ RunReporter) (ExecutionOutcome, error) { + if calls.Add(1) == 1 { + return ExecutionOutcome{}, errors.New("first run failed") + } + started <- request.Run.ID + <-ctx.Done() + return ExecutionOutcome{}, context.Cause(ctx) + }) + manager := newTestManager(t, store, executor) + failed, err := manager.StartRun(context.Background(), definition.ID) + if err != nil { + t.Fatalf("start failing run: %v", err) + } + waitRunStatus(t, store, failed.ID, RunStatusFailed) + blocker, err := manager.StartRun(context.Background(), definition.ID) + if err != nil { + t.Fatalf("start blocking run: %v", err) + } + if got := receiveString(t, started); got != blocker.ID { + t.Fatalf("blocking run = %q, want %q", got, blocker.ID) + } + if _, err := manager.RetryRun(context.Background(), failed.ID); !errors.Is(err, ErrRunAlreadyActive) { + t.Fatalf("retry during active run error = %v, want ErrRunAlreadyActive", err) + } + if err := manager.CancelRun(context.Background(), blocker.ID); err != nil { + t.Fatalf("cancel blocking run: %v", err) + } + waitRunStatus(t, store, blocker.ID, RunStatusCanceled) +} + +func openTestStore(t *testing.T) *Store { + t.Helper() + store, err := Open(t.TempDir() + "/sync-jobs.db") + if err != nil { + t.Fatalf("open store: %v", err) + } + t.Cleanup(func() { + if err := store.Close(); err != nil { + t.Errorf("close store: %v", err) + } + }) + return store +} + +func newTestManager(t *testing.T, store *Store, executor Executor) *Manager { + t.Helper() + manager, err := NewManager(context.Background(), store, executor, ManagerOptions{ + SchedulerInterval: time.Hour, + HeartbeatInterval: time.Hour, + }) + if err != nil { + t.Fatalf("new manager: %v", err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := manager.Shutdown(ctx); err != nil { + t.Errorf("shutdown manager: %v", err) + } + }) + return manager +} + +func newScheduledTestManager(t *testing.T, store *Store, executor Executor, owner string) *Manager { + t.Helper() + manager, err := NewManager(context.Background(), store, executor, ManagerOptions{ + SchedulerInterval: 10 * time.Millisecond, + LeaseTTL: 100 * time.Millisecond, + HeartbeatInterval: time.Hour, + RecoveryStaleAfter: time.Hour, + LeaseOwner: owner, + }) + if err != nil { + t.Fatalf("new scheduled manager: %v", err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := manager.Shutdown(ctx); err != nil { + t.Errorf("shutdown scheduled manager: %v", err) + } + }) + return manager +} + +func putTestJob(t *testing.T, store *Store, concurrencyPolicy string) JobDefinition { + t.Helper() + definition, err := store.PutJob(context.Background(), JobDefinition{ + Name: "orders sync", + Enabled: true, + Kind: JobKindReconcile, + IncrementalMode: IncrementalSnapshot, + Source: EndpointRef{ConnectionID: "source"}, + Target: EndpointRef{ConnectionID: "target"}, + Mappings: []TableMapping{{SourceTable: "orders", TargetTable: "orders", Enabled: true}}, + ConcurrencyPolicy: concurrencyPolicy, + }) + if err != nil { + t.Fatalf("put test job: %v", err) + } + return definition +} + +func receiveString(t *testing.T, values <-chan string) string { + t.Helper() + select { + case value := <-values: + return value + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for value") + return "" + } +} + +func receiveRequest(t *testing.T, values <-chan ExecutionRequest) ExecutionRequest { + t.Helper() + select { + case value := <-values: + return value + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for execution request") + return ExecutionRequest{} + } +} + +func assertRunStatus(t *testing.T, store *Store, runID string, want RunStatus) { + t.Helper() + run, err := store.GetRun(context.Background(), runID) + if err != nil { + t.Fatalf("get run %s: %v", runID, err) + } + if run.Status != want { + t.Fatalf("run %s status = %q, want %q", runID, run.Status, want) + } +} + +func waitRunStatus(t *testing.T, store *Store, runID string, want RunStatus) RunRecord { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + run, err := store.GetRun(context.Background(), runID) + if err == nil && run.Status == want { + return run + } + time.Sleep(5 * time.Millisecond) + } + assertRunStatus(t, store, runID, want) + return RunRecord{} +} + +func waitRunEventCount(t *testing.T, store *Store, runID string, want int) []RunEvent { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + events, err := store.ListRunEvents(context.Background(), runID, 0, want+10) + if err == nil && len(events) >= want { + return events + } + time.Sleep(5 * time.Millisecond) + } + events, err := store.ListRunEvents(context.Background(), runID, 0, want+10) + if err != nil { + t.Fatalf("list run events: %v", err) + } + t.Fatalf("event count = %d, want at least %d: %#v", len(events), want, events) + return nil +} diff --git a/internal/syncjob/model.go b/internal/syncjob/model.go new file mode 100644 index 00000000..49349fa2 --- /dev/null +++ b/internal/syncjob/model.go @@ -0,0 +1,306 @@ +package syncjob + +import "encoding/json" + +const CurrentDefinitionVersion = 1 + +type JobLifecycle string + +const ( + JobLifecycleDraft JobLifecycle = "draft" + JobLifecycleReady JobLifecycle = "ready" + JobLifecycleEnabled JobLifecycle = "enabled" + JobLifecyclePaused JobLifecycle = "paused" + JobLifecycleArchived JobLifecycle = "archived" +) + +type JobKind string + +const ( + JobKindMigration JobKind = "migration" + JobKindReconcile JobKind = "reconcile" + JobKindQuerySink JobKind = "query_sink" + JobKindCompare JobKind = "compare" +) + +type IncrementalMode string + +const ( + IncrementalSnapshot IncrementalMode = "snapshot" + IncrementalWatermark IncrementalMode = "watermark" + IncrementalCDC IncrementalMode = "cdc" +) + +type ScheduleKind string + +const ( + ScheduleManual ScheduleKind = "manual" + ScheduleOnce ScheduleKind = "once" + ScheduleInterval ScheduleKind = "interval" + ScheduleCron ScheduleKind = "cron" + ScheduleContinuous ScheduleKind = "continuous" +) + +type ErrorPolicy string + +const ( + ErrorPolicyStop ErrorPolicy = "stop" + ErrorPolicySkipRow ErrorPolicy = "skip_row" +) + +type EndpointRef struct { + ConnectionID string `json:"connectionId"` + ConnectionType string `json:"connectionType,omitempty"` + ConnectionName string `json:"connectionName,omitempty"` + Database string `json:"database,omitempty"` + Schema string `json:"schema,omitempty"` + Fingerprint string `json:"fingerprint,omitempty"` +} + +type ExecutionApproval struct { + DefinitionHash string `json:"definitionHash"` + TargetFingerprint string `json:"targetFingerprint"` + ApprovedAt int64 `json:"approvedAt"` + ApprovedByRuntime string `json:"approvedByRuntime"` +} + +type TransformSpec struct { + Kind string `json:"kind,omitempty"` + Argument json.RawMessage `json:"argument,omitempty"` +} + +type ColumnMapping struct { + Source string `json:"source,omitempty"` + Target string `json:"target"` + Transform TransformSpec `json:"transform,omitempty"` + DefaultValue json.RawMessage `json:"defaultValue,omitempty"` + Required bool `json:"required,omitempty"` +} + +type WatermarkSpec struct { + Column string `json:"column"` + InitialValue json.RawMessage `json:"initialValue,omitempty"` + TieBreakerColumns []string `json:"tieBreakerColumns,omitempty"` +} + +type TableMapping struct { + SourceSchema string `json:"sourceSchema,omitempty"` + SourceTable string `json:"sourceTable"` + TargetSchema string `json:"targetSchema,omitempty"` + TargetTable string `json:"targetTable"` + TargetTableStrategy string `json:"targetTableStrategy,omitempty"` + Filter string `json:"filter,omitempty"` + KeyColumns []string `json:"keyColumns,omitempty"` + Columns []ColumnMapping `json:"columns,omitempty"` + Watermark *WatermarkSpec `json:"watermark,omitempty"` + Enabled bool `json:"enabled"` +} + +type ExecutionOptions struct { + Content string `json:"content,omitempty"` + SyncMode string `json:"syncMode,omitempty"` + TargetTableStrategy string `json:"targetTableStrategy,omitempty"` + AutoAddColumns bool `json:"autoAddColumns,omitempty"` + CreateIndexes bool `json:"createIndexes,omitempty"` + PropagateDeletes bool `json:"propagateDeletes,omitempty"` + BatchSize int `json:"batchSize,omitempty"` + ErrorPolicy ErrorPolicy `json:"errorPolicy,omitempty"` + MaxRetries int `json:"maxRetries,omitempty"` + RetryBackoffMillis int `json:"retryBackoffMillis,omitempty"` + CaptureErrorPayload bool `json:"captureErrorPayload,omitempty"` +} + +type ScheduleSpec struct { + Kind ScheduleKind `json:"kind"` + RunAt int64 `json:"runAt,omitempty"` + IntervalSeconds int64 `json:"intervalSeconds,omitempty"` + CronExpression string `json:"cronExpression,omitempty"` + Timezone string `json:"timezone,omitempty"` + AnchorAt int64 `json:"anchorAt,omitempty"` + MisfirePolicy string `json:"misfirePolicy,omitempty"` +} + +type CDCSpec struct { + Adapter string `json:"adapter,omitempty"` + StartPosition string `json:"startPosition,omitempty"` + InitialSnapshot bool `json:"initialSnapshot,omitempty"` + SlotName string `json:"slotName,omitempty"` + PublicationName string `json:"publicationName,omitempty"` +} + +type JobDefinition struct { + Version int `json:"version"` + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + Lifecycle JobLifecycle `json:"lifecycle"` + Enabled bool `json:"enabled"` + Kind JobKind `json:"kind"` + IncrementalMode IncrementalMode `json:"incrementalMode"` + Source EndpointRef `json:"source"` + Target EndpointRef `json:"target"` + SourceQuery string `json:"sourceQuery,omitempty"` + Mappings []TableMapping `json:"mappings"` + Options ExecutionOptions `json:"options"` + Schedule ScheduleSpec `json:"schedule"` + CDC *CDCSpec `json:"cdc,omitempty"` + Approval *ExecutionApproval `json:"approval,omitempty"` + ConcurrencyPolicy string `json:"concurrencyPolicy,omitempty"` + ResumePolicy string `json:"resumePolicy,omitempty"` + Revision int64 `json:"revision"` + CreatedAt int64 `json:"createdAt"` + UpdatedAt int64 `json:"updatedAt"` + NextRunAt int64 `json:"nextRunAt,omitempty"` + LastScheduledAt int64 `json:"lastScheduledAt,omitempty"` + ArchivedAt int64 `json:"archivedAt,omitempty"` +} + +type RunStatus string + +const ( + RunStatusQueued RunStatus = "queued" + RunStatusRunning RunStatus = "running" + RunStatusCancelling RunStatus = "cancelling" + RunStatusPaused RunStatus = "paused" + RunStatusSucceeded RunStatus = "succeeded" + RunStatusPartial RunStatus = "partial" + RunStatusFailed RunStatus = "failed" + RunStatusCanceled RunStatus = "canceled" + RunStatusInterrupted RunStatus = "interrupted" +) + +type RunTrigger string + +const ( + RunTriggerManual RunTrigger = "manual" + RunTriggerSchedule RunTrigger = "schedule" + RunTriggerResume RunTrigger = "resume" + RunTriggerRetry RunTrigger = "retry" +) + +type RunRecord struct { + ID string `json:"id"` + JobID string `json:"jobId"` + OwnerToken string `json:"-"` + JobRevision int64 `json:"jobRevision"` + Trigger RunTrigger `json:"trigger"` + Status RunStatus `json:"status"` + ParentRunID string `json:"parentRunId,omitempty"` + Attempt int `json:"attempt"` + QueuedAt int64 `json:"queuedAt"` + StartedAt int64 `json:"startedAt,omitempty"` + FinishedAt int64 `json:"finishedAt,omitempty"` + HeartbeatAt int64 `json:"heartbeatAt,omitempty"` + Current int `json:"current"` + Total int `json:"total"` + Table string `json:"table,omitempty"` + Stage string `json:"stage,omitempty"` + RowsInserted int64 `json:"rowsInserted"` + RowsUpdated int64 `json:"rowsUpdated"` + RowsDeleted int64 `json:"rowsDeleted"` + RowsFailed int64 `json:"rowsFailed"` + Message string `json:"message,omitempty"` + Resumable bool `json:"resumable"` + DefinitionSnapshot json.RawMessage `json:"definitionSnapshot,omitempty"` + SourceFingerprint string `json:"sourceFingerprint,omitempty"` + TargetFingerprint string `json:"targetFingerprint,omitempty"` + CreatedAt int64 `json:"createdAt"` + UpdatedAt int64 `json:"updatedAt"` +} + +type Checkpoint struct { + Version int `json:"version"` + Kind string `json:"kind"` + JobID string `json:"jobId"` + RunID string `json:"runId"` + DefinitionRevision int64 `json:"definitionRevision"` + Table string `json:"table"` + Phase string `json:"phase"` + CursorType string `json:"cursorType"` + Cursor json.RawMessage `json:"cursor,omitempty"` + Watermark json.RawMessage `json:"watermark,omitempty"` + BatchSequence int64 `json:"batchSequence"` + SchemaHash string `json:"schemaHash,omitempty"` + UpdatedAt int64 `json:"updatedAt"` +} + +type ErrorRowStatus string + +const ( + ErrorRowPending ErrorRowStatus = "pending" + ErrorRowRetrying ErrorRowStatus = "retrying" + ErrorRowResolved ErrorRowStatus = "resolved" + ErrorRowDiscarded ErrorRowStatus = "discarded" +) + +type ErrorRow struct { + ID string `json:"id"` + RunID string `json:"runId"` + JobID string `json:"jobId"` + SourceTable string `json:"sourceTable,omitempty"` + TargetTable string `json:"targetTable,omitempty"` + Operation string `json:"operation,omitempty"` + SourceKey json.RawMessage `json:"sourceKey,omitempty"` + Payload json.RawMessage `json:"payload,omitempty"` + PayloadPolicy string `json:"payloadPolicy,omitempty"` + PayloadHash string `json:"payloadHash,omitempty"` + PayloadSize int64 `json:"payloadSize,omitempty"` + Error string `json:"error"` + ErrorCode string `json:"errorCode,omitempty"` + ErrorClass string `json:"errorClass,omitempty"` + Attempts int `json:"attempts"` + Status ErrorRowStatus `json:"status"` + RetryOwner string `json:"-"` + RetryLeaseExpiresAt int64 `json:"-"` + CreatedAt int64 `json:"createdAt"` + UpdatedAt int64 `json:"updatedAt"` +} + +type ExecutionOutcome struct { + RowsInserted int64 `json:"rowsInserted"` + RowsUpdated int64 `json:"rowsUpdated"` + RowsDeleted int64 `json:"rowsDeleted"` + RowsFailed int64 `json:"rowsFailed"` + Message string `json:"message,omitempty"` + Resumable bool `json:"resumable"` +} + +type RunProgress struct { + Current int `json:"current"` + Total int `json:"total"` + Table string `json:"table,omitempty"` + Stage string `json:"stage,omitempty"` + Message string `json:"message,omitempty"` +} + +type RunEventType string + +const ( + RunEventQueued RunEventType = "queued" + RunEventStarted RunEventType = "started" + RunEventProgress RunEventType = "progress" + RunEventCheckpoint RunEventType = "checkpoint" + RunEventErrorRow RunEventType = "error_row" + RunEventLog RunEventType = "log" + RunEventCancelling RunEventType = "cancelling" + RunEventCanceled RunEventType = "canceled" + RunEventSucceeded RunEventType = "succeeded" + RunEventPartial RunEventType = "partial" + RunEventFailed RunEventType = "failed" + RunEventInterrupted RunEventType = "interrupted" +) + +type RunEvent struct { + RunID string `json:"runId"` + JobID string `json:"jobId"` + Sequence int64 `json:"sequence"` + Type RunEventType `json:"type"` + Status RunStatus `json:"status,omitempty"` + Current int `json:"current,omitempty"` + Total int `json:"total,omitempty"` + Table string `json:"table,omitempty"` + Stage string `json:"stage,omitempty"` + Message string `json:"message,omitempty"` + Payload json.RawMessage `json:"payload,omitempty"` + CreatedAt int64 `json:"createdAt"` +} diff --git a/internal/syncjob/permanent_error.go b/internal/syncjob/permanent_error.go new file mode 100644 index 00000000..0e1494dd --- /dev/null +++ b/internal/syncjob/permanent_error.go @@ -0,0 +1,29 @@ +package syncjob + +// PermanentExecutionError marks an execution failure that cannot be repaired by +// retrying the same persisted task definition. The manager pauses the task only +// when the owning executor successfully commits this run's failed terminal state. +type PermanentExecutionError struct { + Err error +} + +func (e *PermanentExecutionError) Error() string { + if e == nil || e.Err == nil { + return "permanent data sync execution failure" + } + return e.Err.Error() +} + +func (e *PermanentExecutionError) Unwrap() error { + if e == nil { + return nil + } + return e.Err +} + +func MarkPermanentExecutionError(err error) error { + if err == nil { + return nil + } + return &PermanentExecutionError{Err: err} +} diff --git a/internal/syncjob/plan_hash.go b/internal/syncjob/plan_hash.go new file mode 100644 index 00000000..b23d401a --- /dev/null +++ b/internal/syncjob/plan_hash.go @@ -0,0 +1,63 @@ +package syncjob + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" +) + +// ExecutionPlanHash identifies the data-affecting portion of a definition. +// Task labels, lifecycle, scheduling and revisions deliberately do not move a +// durable cursor; endpoints, mappings, write policy and incremental scope do. +func ExecutionPlanHash(input JobDefinition) (string, error) { + definition := NormalizeDefinition(input) + definition.ID = "" + definition.Name = "" + definition.Description = "" + definition.Lifecycle = "" + definition.Enabled = false + definition.Source.ConnectionName = "" + definition.Target.ConnectionName = "" + definition.Schedule = ScheduleSpec{} + definition.Approval = nil + definition.ConcurrencyPolicy = "" + definition.ResumePolicy = "" + definition.Revision = 0 + definition.CreatedAt = 0 + definition.UpdatedAt = 0 + definition.NextRunAt = 0 + definition.LastScheduledAt = 0 + definition.ArchivedAt = 0 + return hashJobDefinition(definition) +} + +// ApprovalScopeHash identifies the exact operation a user approved. Unlike +// ExecutionPlanHash it deliberately retains task identity, lifecycle, +// scheduling, concurrency and resume policy: approving a manual run must not +// authorize turning the same data plan into an unattended continuous job. +// Only presentation fields, persistence metadata and the approval itself are +// excluded. +func ApprovalScopeHash(input JobDefinition) (string, error) { + definition := NormalizeDefinition(input) + definition.Name = "" + definition.Description = "" + definition.Source.ConnectionName = "" + definition.Target.ConnectionName = "" + definition.Approval = nil + definition.Revision = 0 + definition.CreatedAt = 0 + definition.UpdatedAt = 0 + definition.NextRunAt = 0 + definition.LastScheduledAt = 0 + definition.ArchivedAt = 0 + return hashJobDefinition(definition) +} + +func hashJobDefinition(definition JobDefinition) (string, error) { + payload, err := json.Marshal(definition) + if err != nil { + return "", err + } + sum := sha256.Sum256(payload) + return "sha256:" + hex.EncodeToString(sum[:]), nil +} diff --git a/internal/syncjob/plan_hash_test.go b/internal/syncjob/plan_hash_test.go new file mode 100644 index 00000000..318b58e8 --- /dev/null +++ b/internal/syncjob/plan_hash_test.go @@ -0,0 +1,90 @@ +package syncjob + +import "testing" + +func TestExecutionPlanHashIgnoresTaskMetadataButTracksDataSemantics(t *testing.T) { + definition := validValidationTestDefinition() + definition.ID = "job-1" + definition.Revision = 4 + definition.Source.Fingerprint = "source-fingerprint" + definition.Target.Fingerprint = "target-fingerprint" + base, err := ExecutionPlanHash(definition) + if err != nil { + t.Fatalf("hash base plan: %v", err) + } + + metadata := definition + metadata.Name = "renamed" + metadata.Description = "new description" + metadata.Lifecycle = JobLifecyclePaused + metadata.Enabled = false + metadata.Schedule = ScheduleSpec{Kind: ScheduleCron, CronExpression: "0 1 * * *", Timezone: "UTC"} + metadata.Revision++ + got, err := ExecutionPlanHash(metadata) + if err != nil { + t.Fatalf("hash metadata plan: %v", err) + } + if got != base { + t.Fatal("task metadata unexpectedly invalidated the execution plan") + } + + changed := definition + changed.Mappings[0].TargetTable = "orders_v2" + got, err = ExecutionPlanHash(changed) + if err != nil { + t.Fatalf("hash changed plan: %v", err) + } + if got == base { + t.Fatal("target mapping change did not invalidate the execution plan") + } +} + +func TestApprovalScopeHashTracksTaskAndUnattendedExecutionPolicy(t *testing.T) { + definition := validValidationTestDefinition() + definition.ID = "job-1" + definition.Lifecycle = JobLifecycleReady + definition.Enabled = false + definition.Schedule = ScheduleSpec{Kind: ScheduleManual} + base, err := ApprovalScopeHash(definition) + if err != nil { + t.Fatalf("hash approval scope: %v", err) + } + + metadata := definition + metadata.Name = "renamed" + metadata.Description = "new description" + metadata.Revision = 99 + metadata.UpdatedAt = 1234 + got, err := ApprovalScopeHash(metadata) + if err != nil { + t.Fatalf("hash approval metadata: %v", err) + } + if got != base { + t.Fatal("presentation or persistence metadata unexpectedly invalidated approval") + } + + for name, mutate := range map[string]func(*JobDefinition){ + "task identity": func(value *JobDefinition) { value.ID = "job-2" }, + "lifecycle": func(value *JobDefinition) { + value.Lifecycle = JobLifecycleEnabled + value.Enabled = true + }, + "schedule": func(value *JobDefinition) { + value.Schedule = ScheduleSpec{Kind: ScheduleContinuous} + }, + "concurrency": func(value *JobDefinition) { value.ConcurrencyPolicy = "queue" }, + "resume": func(value *JobDefinition) { value.ResumePolicy = "auto" }, + } { + t.Run(name, func(t *testing.T) { + changed := definition + mutate(&changed) + got, hashErr := ApprovalScopeHash(changed) + if hashErr != nil { + t.Fatalf("hash changed approval scope: %v", hashErr) + } + if got == base { + t.Fatalf("%s change did not invalidate approval", name) + } + }) + } +} diff --git a/internal/syncjob/runtime_safety_test.go b/internal/syncjob/runtime_safety_test.go new file mode 100644 index 00000000..78064c06 --- /dev/null +++ b/internal/syncjob/runtime_safety_test.go @@ -0,0 +1,673 @@ +package syncjob + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "sync/atomic" + "testing" + "time" +) + +func TestStoreClaimRunRequiresRunnableJobLifecycle(t *testing.T) { + store := openTestStore(t) + + enabled := putTestJob(t, store, "queue") + ready := putLifecycleTestJob(t, store, "ready", JobLifecycleReady) + draft, err := store.PutJob(context.Background(), JobDefinition{Name: "draft", Lifecycle: JobLifecycleDraft}) + if err != nil { + t.Fatalf("put draft job: %v", err) + } + paused := putLifecycleTestJob(t, store, "paused", JobLifecycleReady) + paused, err = store.PauseJob(context.Background(), paused.ID) + if err != nil { + t.Fatalf("pause job: %v", err) + } + archived := putLifecycleTestJob(t, store, "archived", JobLifecycleReady) + if err := store.DeleteJob(context.Background(), archived.ID); err != nil { + t.Fatalf("archive job: %v", err) + } + archived, err = store.GetJob(context.Background(), archived.ID) + if err != nil { + t.Fatalf("get archived job: %v", err) + } + + for _, test := range []struct { + name string + job JobDefinition + claimable bool + }{ + {name: "enabled", job: enabled, claimable: true}, + {name: "ready", job: ready, claimable: true}, + {name: "draft", job: draft, claimable: false}, + {name: "paused", job: paused, claimable: false}, + {name: "archived", job: archived, claimable: false}, + } { + t.Run(test.name, func(t *testing.T) { + run := createStoredRun(t, store, test.job, RunStatusQueued) + claimed, ok, err := store.ClaimRun(context.Background(), run.ID, time.Now().UnixMilli()) + if err != nil { + t.Fatalf("claim run: %v", err) + } + if ok != test.claimable { + t.Fatalf("claimed = %v, want %v", ok, test.claimable) + } + if ok && claimed.OwnerToken == "" { + t.Fatal("claimed run has no fencing token") + } + }) + } +} + +func TestManagerArchiveCancelsQueuedAndRunningRuns(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "queue") + started := make(chan string, 2) + exited := make(chan struct{}, 1) + executor := ExecutorFunc(func(ctx context.Context, request ExecutionRequest, _ RunReporter) (ExecutionOutcome, error) { + started <- request.Run.ID + <-ctx.Done() + exited <- struct{}{} + return ExecutionOutcome{}, context.Cause(ctx) + }) + manager := newTestManager(t, store, executor) + + running, err := manager.StartRun(context.Background(), definition.ID) + if err != nil { + t.Fatalf("start running run: %v", err) + } + if got := receiveString(t, started); got != running.ID { + t.Fatalf("started run = %s, want %s", got, running.ID) + } + queued, err := manager.StartRun(context.Background(), definition.ID) + if err != nil { + t.Fatalf("start queued run: %v", err) + } + if err := manager.DeleteJob(context.Background(), definition.ID); err != nil { + t.Fatalf("archive job: %v", err) + } + waitRunStatus(t, store, queued.ID, RunStatusCanceled) + waitRunStatus(t, store, running.ID, RunStatusCanceled) + select { + case <-exited: + case <-time.After(time.Second): + t.Fatal("running executor did not observe archive cancellation") + } + select { + case got := <-started: + t.Fatalf("archived queued run unexpectedly executed: %s", got) + case <-time.After(75 * time.Millisecond): + } + archived, err := store.GetJob(context.Background(), definition.ID) + if err != nil || archived.Lifecycle != JobLifecycleArchived || archived.Enabled { + t.Fatalf("archived definition = %#v, err=%v", archived, err) + } +} + +func TestManagerPauseAndPausedPutCancelActiveRuns(t *testing.T) { + store := openTestStore(t) + first := putTestJob(t, store, "queue") + second := putTestJob(t, store, "queue") + started := make(chan string, 2) + executor := ExecutorFunc(func(ctx context.Context, request ExecutionRequest, _ RunReporter) (ExecutionOutcome, error) { + started <- request.Run.JobID + <-ctx.Done() + return ExecutionOutcome{}, context.Cause(ctx) + }) + manager, err := NewManager(context.Background(), store, executor, ManagerOptions{ + SchedulerInterval: time.Hour, + HeartbeatInterval: time.Hour, + MaxConcurrentRuns: 2, + }) + if err != nil { + t.Fatalf("new manager: %v", err) + } + t.Cleanup(func() { shutdownTestManager(t, manager) }) + + firstRun, err := manager.StartRun(context.Background(), first.ID) + if err != nil { + t.Fatalf("start first run: %v", err) + } + secondRun, err := manager.StartRun(context.Background(), second.ID) + if err != nil { + t.Fatalf("start second run: %v", err) + } + seen := map[string]bool{receiveString(t, started): true, receiveString(t, started): true} + if !seen[first.ID] || !seen[second.ID] { + t.Fatalf("started jobs = %#v", seen) + } + paused, err := manager.PauseJob(context.Background(), first.ID) + if err != nil || paused.Lifecycle != JobLifecyclePaused || paused.Enabled { + t.Fatalf("pause job = %#v, err=%v", paused, err) + } + second.Lifecycle = JobLifecyclePaused + second.Enabled = false + pausedByPut, err := manager.PutJob(context.Background(), second) + if err != nil || pausedByPut.Lifecycle != JobLifecyclePaused || pausedByPut.Enabled { + t.Fatalf("put paused job = %#v, err=%v", pausedByPut, err) + } + waitRunStatus(t, store, firstRun.ID, RunStatusCanceled) + waitRunStatus(t, store, secondRun.ID, RunStatusCanceled) +} + +func TestManagerEnforcesMaximumConcurrentRunsAcrossJobs(t *testing.T) { + store := openTestStore(t) + jobs := make([]JobDefinition, 0, 5) + for index := 0; index < 5; index++ { + jobs = append(jobs, putTestJob(t, store, "queue")) + } + started := make(chan string, len(jobs)) + release := make(chan struct{}, len(jobs)) + var active atomic.Int32 + var maximum atomic.Int32 + executor := ExecutorFunc(func(ctx context.Context, request ExecutionRequest, _ RunReporter) (ExecutionOutcome, error) { + current := active.Add(1) + defer active.Add(-1) + for { + seen := maximum.Load() + if current <= seen || maximum.CompareAndSwap(seen, current) { + break + } + } + started <- request.Run.ID + select { + case <-ctx.Done(): + return ExecutionOutcome{}, context.Cause(ctx) + case <-release: + return ExecutionOutcome{}, nil + } + }) + manager, err := NewManager(context.Background(), store, executor, ManagerOptions{ + SchedulerInterval: time.Hour, + HeartbeatInterval: time.Hour, + MaxConcurrentRuns: 2, + }) + if err != nil { + t.Fatalf("new manager: %v", err) + } + t.Cleanup(func() { shutdownTestManager(t, manager) }) + runs := make([]RunRecord, 0, len(jobs)) + for _, job := range jobs { + run, err := manager.StartRun(context.Background(), job.ID) + if err != nil { + t.Fatalf("start run: %v", err) + } + runs = append(runs, run) + } + _ = receiveString(t, started) + _ = receiveString(t, started) + select { + case runID := <-started: + t.Fatalf("third run exceeded concurrency limit: %s", runID) + case <-time.After(75 * time.Millisecond): + } + for index := 0; index < len(jobs); index++ { + release <- struct{}{} + } + for _, run := range runs { + waitRunStatus(t, store, run.ID, RunStatusSucceeded) + } + if got := maximum.Load(); got != 2 { + t.Fatalf("maximum concurrent runs = %d, want 2", got) + } +} + +func TestStoreRunOwnershipFencesStaleExecutorMutations(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "queue") + first := createStoredRun(t, store, definition, RunStatusQueued) + first, claimed, err := store.ClaimRun(context.Background(), first.ID, time.Now().UnixMilli()) + if err != nil || !claimed || first.OwnerToken == "" { + t.Fatalf("claim first run = %#v, claimed=%v, err=%v", first, claimed, err) + } + if _, err := store.PutCheckpointOwned(context.Background(), testCheckpoint(definition, first, 1), first.OwnerToken); err != nil { + t.Fatalf("save first checkpoint: %v", err) + } + recovered, err := store.InterruptStaleRuns(context.Background(), first.HeartbeatAt+1, first.HeartbeatAt+2) + if err != nil || len(recovered) != 1 || recovered[0].ID != first.ID { + t.Fatalf("recover first run = %#v, err=%v", recovered, err) + } + + second := createStoredRun(t, store, definition, RunStatusQueued) + second, claimed, err = store.ClaimRun(context.Background(), second.ID, first.HeartbeatAt+3) + if err != nil || !claimed || second.OwnerToken == "" || second.OwnerToken == first.OwnerToken { + t.Fatalf("claim second run = %#v, claimed=%v, err=%v", second, claimed, err) + } + if _, err := store.PutCheckpointOwned(context.Background(), testCheckpoint(definition, second, 2), second.OwnerToken); err != nil { + t.Fatalf("save second checkpoint: %v", err) + } + if err := store.TouchRun(context.Background(), second.ID, first.HeartbeatAt+4); !errors.Is(err, ErrRunOwnershipLost) { + t.Fatalf("unowned heartbeat bypass error = %v, want ErrRunOwnershipLost", err) + } + if _, err := store.PutCheckpoint(context.Background(), testCheckpoint(definition, second, 3)); !errors.Is(err, ErrRunOwnershipLost) { + t.Fatalf("unowned checkpoint bypass error = %v, want ErrRunOwnershipLost", err) + } + if err := store.DeleteCheckpoint(context.Background(), definition.ID); !errors.Is(err, ErrRunOwnershipLost) { + t.Fatalf("unowned checkpoint delete bypass error = %v, want ErrRunOwnershipLost", err) + } + if _, err := store.CompleteRun(context.Background(), second.ID, RunStatusSucceeded, ExecutionOutcome{}, "unowned success", first.HeartbeatAt+4); !errors.Is(err, ErrRunOwnershipLost) { + t.Fatalf("unowned completion bypass error = %v, want ErrRunOwnershipLost", err) + } + + if err := store.TouchRunOwned(context.Background(), first.ID, first.OwnerToken, first.HeartbeatAt+4); !errors.Is(err, ErrRunOwnershipLost) { + t.Fatalf("stale heartbeat error = %v, want ErrRunOwnershipLost", err) + } + if _, err := store.UpdateRunProgressOwned(context.Background(), first.ID, first.OwnerToken, RunProgress{Current: 1, Total: 1}, first.HeartbeatAt+4); !errors.Is(err, ErrRunOwnershipLost) { + t.Fatalf("stale progress error = %v, want ErrRunOwnershipLost", err) + } + if _, err := store.PutCheckpointOwned(context.Background(), testCheckpoint(definition, first, 99), first.OwnerToken); !errors.Is(err, ErrRunOwnershipLost) { + t.Fatalf("stale checkpoint error = %v, want ErrRunOwnershipLost", err) + } + if err := store.DeleteCheckpointOwned(context.Background(), definition.ID, first.ID, first.OwnerToken); !errors.Is(err, ErrRunOwnershipLost) { + t.Fatalf("stale checkpoint delete error = %v, want ErrRunOwnershipLost", err) + } + if _, err := store.CompleteRunOwned(context.Background(), first.ID, first.OwnerToken, RunStatusSucceeded, ExecutionOutcome{}, "stale success", first.HeartbeatAt+4); !errors.Is(err, ErrRunOwnershipLost) { + t.Fatalf("stale completion error = %v, want ErrRunOwnershipLost", err) + } + checkpoint, err := store.GetCheckpoint(context.Background(), definition.ID) + if err != nil || checkpoint.RunID != second.ID || checkpoint.BatchSequence != 2 { + t.Fatalf("checkpoint after stale writes = %#v, err=%v", checkpoint, err) + } + if _, err := store.CompleteRunOwned(context.Background(), second.ID, second.OwnerToken, RunStatusSucceeded, ExecutionOutcome{}, "", first.HeartbeatAt+5); err != nil { + t.Fatalf("complete second run: %v", err) + } +} + +func TestManagerHeartbeatCancelsExecutorAfterOwnershipLoss(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "queue") + started := make(chan string, 1) + exited := make(chan error, 1) + executor := ExecutorFunc(func(ctx context.Context, request ExecutionRequest, _ RunReporter) (ExecutionOutcome, error) { + started <- request.Run.ID + <-ctx.Done() + exited <- context.Cause(ctx) + return ExecutionOutcome{}, nil + }) + manager, err := NewManager(context.Background(), store, executor, ManagerOptions{ + SchedulerInterval: time.Hour, + HeartbeatInterval: 10 * time.Millisecond, + RecoveryStaleAfter: time.Hour, + }) + if err != nil { + t.Fatalf("new manager: %v", err) + } + t.Cleanup(func() { shutdownTestManager(t, manager) }) + run, err := manager.StartRun(context.Background(), definition.ID) + if err != nil { + t.Fatalf("start run: %v", err) + } + if got := receiveString(t, started); got != run.ID { + t.Fatalf("started run = %s, want %s", got, run.ID) + } + if _, err := store.db.ExecContext(context.Background(), `UPDATE data_sync_runs SET owner_token = 'replacement-owner' WHERE id = ?`, run.ID); err != nil { + t.Fatalf("replace run owner: %v", err) + } + select { + case cause := <-exited: + if !errors.Is(cause, ErrRunOwnershipLost) { + t.Fatalf("executor cancellation cause = %v, want ErrRunOwnershipLost", cause) + } + case <-time.After(time.Second): + t.Fatal("executor was not canceled after ownership loss") + } + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + manager.mu.Lock() + _, active := manager.active[run.ID] + manager.mu.Unlock() + if !active { + break + } + time.Sleep(time.Millisecond) + } + persisted, err := store.GetRun(context.Background(), run.ID) + if err != nil { + t.Fatalf("get fenced run: %v", err) + } + if persisted.Status != RunStatusRunning || persisted.OwnerToken != "replacement-owner" { + t.Fatalf("stale executor overwrote fenced run: %#v", persisted) + } +} + +func TestStandbyLeaseHolderPeriodicallyRecoversStaleRuns(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "queue") + run := createStoredRun(t, store, definition, RunStatusQueued) + run, claimed, err := store.ClaimRun(context.Background(), run.ID, time.Now().UnixMilli()) + if err != nil || !claimed { + t.Fatalf("claim run: claimed=%v, err=%v", claimed, err) + } + now := time.Now() + if acquired, err := store.AcquireSchedulerLease(context.Background(), "data-sync-scheduler", "primary", now, 100*time.Millisecond); err != nil || !acquired { + t.Fatalf("prime scheduler lease: acquired=%v, err=%v", acquired, err) + } + executor := ExecutorFunc(func(context.Context, ExecutionRequest, RunReporter) (ExecutionOutcome, error) { + t.Fatal("recovered direct run must not execute") + return ExecutionOutcome{}, nil + }) + primary, err := NewManager(context.Background(), store, executor, ManagerOptions{ + SchedulerInterval: 10 * time.Millisecond, + LeaseTTL: 100 * time.Millisecond, + HeartbeatInterval: time.Hour, + RecoveryStaleAfter: 120 * time.Millisecond, + RecoveryInterval: 20 * time.Millisecond, + LeaseOwner: "primary", + }) + if err != nil { + t.Fatalf("new primary manager: %v", err) + } + standby, err := NewManager(context.Background(), store, executor, ManagerOptions{ + SchedulerInterval: 10 * time.Millisecond, + LeaseTTL: 100 * time.Millisecond, + HeartbeatInterval: time.Hour, + RecoveryStaleAfter: 120 * time.Millisecond, + RecoveryInterval: 20 * time.Millisecond, + LeaseOwner: "standby", + }) + if err != nil { + shutdownTestManager(t, primary) + t.Fatalf("new standby manager: %v", err) + } + t.Cleanup(func() { shutdownTestManager(t, standby) }) + shutdownTestManager(t, primary) + recovered := waitRunStatus(t, store, run.ID, RunStatusInterrupted) + if !recovered.Resumable { + t.Fatalf("recovered run is not resumable: %#v", recovered) + } +} + +func TestManagerRestartFinalizesStaleCancellingWithoutAutoResume(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "forbid") + definition.ResumePolicy = "auto" + definition, err := store.PutJob(context.Background(), definition) + if err != nil { + t.Fatalf("enable auto resume: %v", err) + } + snapshot, err := json.Marshal(definition) + if err != nil { + t.Fatalf("marshal definition: %v", err) + } + staleAt := time.Now().Add(-time.Hour).UnixMilli() + stale, err := store.CreateRun(context.Background(), RunRecord{ + JobID: definition.ID, + JobRevision: definition.Revision, + Status: RunStatusRunning, + StartedAt: staleAt, + HeartbeatAt: staleAt, + DefinitionSnapshot: snapshot, + }) + if err != nil { + t.Fatalf("create stale run: %v", err) + } + if _, err := store.PutCheckpoint(context.Background(), testCheckpoint(definition, stale, 1)); err != nil { + t.Fatalf("put stale checkpoint: %v", err) + } + if _, err := store.RequestCancelRun(context.Background(), stale.ID, staleAt+1); err != nil { + t.Fatalf("request stale cancellation: %v", err) + } + + var executions atomic.Int32 + manager, err := NewManager(context.Background(), store, ExecutorFunc(func(context.Context, ExecutionRequest, RunReporter) (ExecutionOutcome, error) { + executions.Add(1) + return ExecutionOutcome{}, nil + }), ManagerOptions{ + SchedulerInterval: time.Hour, + HeartbeatInterval: time.Hour, + RecoveryStaleAfter: time.Second, + }) + if err != nil { + t.Fatalf("restart manager: %v", err) + } + t.Cleanup(func() { shutdownTestManager(t, manager) }) + + recovered, err := store.GetRun(context.Background(), stale.ID) + if err != nil { + t.Fatalf("get recovered cancellation: %v", err) + } + if recovered.Status != RunStatusCanceled || recovered.Resumable || recovered.OwnerToken != "" { + t.Fatalf("recovered cancellation = %#v", recovered) + } + events, err := store.ListRunEvents(context.Background(), stale.ID, 0, 10) + if err != nil { + t.Fatalf("list recovery events: %v", err) + } + if len(events) != 1 || events[0].Type != RunEventCanceled { + t.Fatalf("recovery events = %#v", events) + } + time.Sleep(50 * time.Millisecond) + runs, err := store.ListRuns(context.Background(), definition.ID, 10) + if err != nil { + t.Fatalf("list runs after recovery: %v", err) + } + if len(runs) != 1 || executions.Load() != 0 { + t.Fatalf("cancellation resumed after restart: runs=%#v executions=%d", runs, executions.Load()) + } +} + +func TestManagerResumeAcceptsCheckpointFromAncestorRun(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "forbid") + requests := make(chan ExecutionRequest, 3) + var calls atomic.Int32 + executor := ExecutorFunc(func(_ context.Context, request ExecutionRequest, reporter RunReporter) (ExecutionOutcome, error) { + requests <- request + switch calls.Add(1) { + case 1: + if err := reporter.SaveCheckpoint(Checkpoint{ + Kind: "watermark", Table: "orders", Phase: "copy", CursorType: "primary_key", Cursor: json.RawMessage(`{"id":1}`), + }); err != nil { + return ExecutionOutcome{}, err + } + return ExecutionOutcome{Resumable: true}, errors.New("first failure") + case 2: + return ExecutionOutcome{Resumable: true}, errors.New("resume failed before checkpoint") + default: + return ExecutionOutcome{}, nil + } + }) + manager := newTestManager(t, store, executor) + first, err := manager.StartRun(context.Background(), definition.ID) + if err != nil { + t.Fatalf("start first run: %v", err) + } + firstRequest := receiveRequest(t, requests) + if firstRequest.Run.ID != first.ID { + t.Fatalf("first request run = %s, want %s", firstRequest.Run.ID, first.ID) + } + waitRunStatus(t, store, first.ID, RunStatusFailed) + second, err := manager.ResumeRun(context.Background(), first.ID) + if err != nil { + t.Fatalf("resume first run: %v", err) + } + secondRequest := receiveRequest(t, requests) + if secondRequest.Run.ID != second.ID || secondRequest.Checkpoint == nil || secondRequest.Checkpoint.RunID != first.ID { + t.Fatalf("second request = %#v", secondRequest) + } + waitRunStatus(t, store, second.ID, RunStatusFailed) + third, err := manager.ResumeRun(context.Background(), second.ID) + if err != nil { + t.Fatalf("resume second run from ancestor checkpoint: %v", err) + } + thirdRequest := receiveRequest(t, requests) + if thirdRequest.Run.ID != third.ID || thirdRequest.Checkpoint == nil || thirdRequest.Checkpoint.RunID != first.ID { + t.Fatalf("third request = %#v", thirdRequest) + } + waitRunStatus(t, store, third.ID, RunStatusSucceeded) +} + +func TestManagerRejectsResumeAndRetryForInsertOnlyRuns(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "forbid") + definition.Options.SyncMode = "insert_only" + definition, err := store.PutJob(context.Background(), definition) + if err != nil { + t.Fatalf("enable insert-only mode: %v", err) + } + executor := ExecutorFunc(func(_ context.Context, _ ExecutionRequest, reporter RunReporter) (ExecutionOutcome, error) { + if err := reporter.SaveCheckpoint(Checkpoint{ + Kind: "watermark", Table: "orders", Phase: "copy", CursorType: "primary_key", Cursor: json.RawMessage(`{"id":1}`), + }); err != nil { + return ExecutionOutcome{}, err + } + return ExecutionOutcome{Resumable: true}, errors.New("partial insert-only failure") + }) + manager := newTestManager(t, store, executor) + run, err := manager.StartRun(context.Background(), definition.ID) + if err != nil { + t.Fatalf("start insert-only run: %v", err) + } + waitRunStatus(t, store, run.ID, RunStatusFailed) + if _, err := manager.ResumeRun(context.Background(), run.ID); !errors.Is(err, ErrRunNotResumable) { + t.Fatalf("insert-only resume error = %v, want ErrRunNotResumable", err) + } + if _, err := manager.RetryRun(context.Background(), run.ID); !errors.Is(err, ErrRunNotRetryable) { + t.Fatalf("insert-only retry error = %v, want ErrRunNotRetryable", err) + } +} + +func TestStoreMigratesRunOwnershipColumnFromVersionTwo(t *testing.T) { + path := t.TempDir() + "/sync-jobs.db" + store, err := Open(path) + if err != nil { + t.Fatalf("create current store: %v", err) + } + if err := store.Close(); err != nil { + t.Fatalf("close current store: %v", err) + } + database, err := sql.Open("sqlite", sqliteDSN(path)) + if err != nil { + t.Fatalf("open raw store: %v", err) + } + if _, err := database.Exec(`ALTER TABLE data_sync_runs DROP COLUMN owner_token`); err != nil { + _ = database.Close() + t.Fatalf("remove ownership column: %v", err) + } + if _, err := database.Exec(`PRAGMA user_version=2`); err != nil { + _ = database.Close() + t.Fatalf("downgrade schema marker: %v", err) + } + if err := database.Close(); err != nil { + t.Fatalf("close raw store: %v", err) + } + migrated, err := Open(path) + if err != nil { + t.Fatalf("migrate version two store: %v", err) + } + t.Cleanup(func() { _ = migrated.Close() }) + hasColumn, err := sqliteTableHasColumn(context.Background(), migrated.db, "data_sync_runs", "owner_token") + if err != nil || !hasColumn { + t.Fatalf("owner_token column present = %v, err=%v", hasColumn, err) + } +} + +func TestStoreMigratesErrorRowRetryLeaseFromVersionThree(t *testing.T) { + path := t.TempDir() + "/sync-jobs.db" + store, err := Open(path) + if err != nil { + t.Fatalf("create current store: %v", err) + } + if err := store.Close(); err != nil { + t.Fatalf("close current store: %v", err) + } + database, err := sql.Open("sqlite", sqliteDSN(path)) + if err != nil { + t.Fatalf("open raw store: %v", err) + } + for _, statement := range []string{ + `DROP INDEX idx_data_sync_error_rows_retry`, + `ALTER TABLE data_sync_error_rows DROP COLUMN retry_owner`, + `ALTER TABLE data_sync_error_rows DROP COLUMN retry_lease_expires_at`, + `PRAGMA user_version=3`, + } { + if _, err := database.Exec(statement); err != nil { + _ = database.Close() + t.Fatalf("prepare version three store (%s): %v", statement, err) + } + } + if err := database.Close(); err != nil { + t.Fatalf("close raw store: %v", err) + } + migrated, err := Open(path) + if err != nil { + t.Fatalf("migrate version three store: %v", err) + } + t.Cleanup(func() { _ = migrated.Close() }) + for _, column := range []string{"retry_owner", "retry_lease_expires_at"} { + hasColumn, err := sqliteTableHasColumn(context.Background(), migrated.db, "data_sync_error_rows", column) + if err != nil || !hasColumn { + t.Fatalf("%s column present = %v, err=%v", column, hasColumn, err) + } + } +} + +func TestManagerRunCreationRollsBackWhenQueuedEventCannotPersist(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "queue") + if _, err := store.db.ExecContext(context.Background(), `CREATE TRIGGER fail_queued_run_event + BEFORE INSERT ON data_sync_run_events WHEN NEW.event_type = 'queued' + BEGIN SELECT RAISE(ABORT, 'injected queued event failure'); END`); err != nil { + t.Fatalf("create event failure trigger: %v", err) + } + manager := newTestManager(t, store, ExecutorFunc(func(context.Context, ExecutionRequest, RunReporter) (ExecutionOutcome, error) { + t.Fatal("run without queued event must not execute") + return ExecutionOutcome{}, nil + })) + if _, err := manager.StartRun(context.Background(), definition.ID); err == nil { + t.Fatal("start run succeeded despite queued event failure") + } + runs, err := store.ListRuns(context.Background(), definition.ID, 10) + if err != nil { + t.Fatalf("list runs: %v", err) + } + if len(runs) != 0 { + t.Fatalf("run persisted without queued event: %#v", runs) + } +} + +func putLifecycleTestJob(t *testing.T, store *Store, name string, lifecycle JobLifecycle) JobDefinition { + t.Helper() + definition, err := store.PutJob(context.Background(), JobDefinition{ + Name: name, Lifecycle: lifecycle, Kind: JobKindReconcile, IncrementalMode: IncrementalSnapshot, + Source: EndpointRef{ConnectionID: "source-" + name}, Target: EndpointRef{ConnectionID: "target-" + name}, + Mappings: []TableMapping{{SourceTable: "orders", TargetTable: "orders", Enabled: true}}, + }) + if err != nil { + t.Fatalf("put %s job: %v", name, err) + } + return definition +} + +func createStoredRun(t *testing.T, store *Store, definition JobDefinition, status RunStatus) RunRecord { + t.Helper() + snapshot, err := json.Marshal(definition) + if err != nil { + t.Fatalf("marshal run definition: %v", err) + } + run, err := store.CreateRun(context.Background(), RunRecord{ + JobID: definition.ID, JobRevision: definition.Revision, Status: status, DefinitionSnapshot: snapshot, + SourceFingerprint: definition.Source.Fingerprint, TargetFingerprint: definition.Target.Fingerprint, + }) + if err != nil { + t.Fatalf("create stored run: %v", err) + } + return run +} + +func testCheckpoint(definition JobDefinition, run RunRecord, sequence int64) Checkpoint { + return Checkpoint{ + Kind: "watermark", JobID: definition.ID, RunID: run.ID, DefinitionRevision: definition.Revision, + Table: "orders", Phase: "copy", CursorType: "primary_key", Cursor: json.RawMessage(`{"id":1}`), BatchSequence: sequence, + } +} + +func shutdownTestManager(t *testing.T, manager *Manager) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := manager.Shutdown(ctx); err != nil { + t.Errorf("shutdown manager: %v", err) + } +} diff --git a/internal/syncjob/store.go b/internal/syncjob/store.go new file mode 100644 index 00000000..2324a514 --- /dev/null +++ b/internal/syncjob/store.go @@ -0,0 +1,1865 @@ +package syncjob + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/google/uuid" + _ "modernc.org/sqlite" +) + +const storeSchemaVersion = 4 + +var ( + ErrClosed = errors.New("data sync job store is closed") + ErrNotFound = errors.New("data sync job record not found") + ErrRevisionConflict = errors.New("data sync job revision conflict") + ErrRunAlreadyActive = errors.New("data sync job already has an unfinished run") + ErrRunNotCancelable = errors.New("data sync run cannot be canceled") + ErrRunOwnershipLost = errors.New("data sync run ownership was lost") + ErrErrorRowStateConflict = errors.New("data sync error row state transition conflict") + ErrErrorRowRetryOwnershipLost = errors.New("data sync error row retry ownership was lost") +) + +type Store struct { + db *sql.DB + path string +} + +func Open(path string) (*Store, error) { + path = strings.TrimSpace(path) + if path == "" { + return nil, errors.New("data sync job database path is empty") + } + absPath, err := filepath.Abs(path) + if err != nil { + return nil, fmt.Errorf("resolve data sync job database path: %w", err) + } + if err := os.MkdirAll(filepath.Dir(absPath), 0o700); err != nil { + return nil, fmt.Errorf("create data sync job directory: %w", err) + } + database, err := sql.Open("sqlite", sqliteDSN(absPath)) + if err != nil { + return nil, fmt.Errorf("open data sync job database: %w", err) + } + database.SetMaxOpenConns(4) + database.SetMaxIdleConns(4) + store := &Store{db: database, path: absPath} + if err := store.initialize(context.Background()); err != nil { + _ = database.Close() + return nil, err + } + if err := os.Chmod(absPath, 0o600); err != nil { + _ = database.Close() + return nil, fmt.Errorf("secure data sync job database: %w", err) + } + return store, nil +} + +func (s *Store) Path() string { + if s == nil { + return "" + } + return s.path +} + +func (s *Store) Close() error { + if s == nil || s.db == nil { + return nil + } + _, checkpointErr := s.db.ExecContext(context.Background(), `PRAGMA wal_checkpoint(TRUNCATE)`) + closeErr := s.db.Close() + s.db = nil + return errors.Join(checkpointErr, closeErr) +} + +func (s *Store) initialize(ctx context.Context) error { + if err := s.ensureOpen(); err != nil { + return err + } + for _, pragma := range []string{ + "PRAGMA busy_timeout=5000", + "PRAGMA journal_mode=WAL", + "PRAGMA synchronous=FULL", + "PRAGMA foreign_keys=ON", + } { + if _, err := s.db.ExecContext(ctx, pragma); err != nil { + return fmt.Errorf("configure data sync job database (%s): %w", pragma, err) + } + } + var version int + if err := s.db.QueryRowContext(ctx, `PRAGMA user_version`).Scan(&version); err != nil { + return fmt.Errorf("read data sync job schema version: %w", err) + } + if version > storeSchemaVersion { + return fmt.Errorf("data sync job schema version %d is newer than supported version %d", version, storeSchemaVersion) + } + statements := []string{ + `CREATE TABLE IF NOT EXISTS data_sync_jobs ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + version INTEGER NOT NULL, + enabled INTEGER NOT NULL, + job_kind TEXT NOT NULL, + incremental_mode TEXT NOT NULL, + source_connection_id TEXT NOT NULL, + target_connection_id TEXT NOT NULL, + definition_json BLOB NOT NULL, + revision INTEGER NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + next_run_at INTEGER NOT NULL DEFAULT 0, + last_scheduled_at INTEGER NOT NULL DEFAULT 0, + archived_at INTEGER NOT NULL DEFAULT 0 + )`, + `CREATE INDEX IF NOT EXISTS idx_data_sync_jobs_schedule ON data_sync_jobs(enabled, next_run_at)`, + `CREATE TABLE IF NOT EXISTS data_sync_runs ( + id TEXT PRIMARY KEY, + job_id TEXT NOT NULL, + job_revision INTEGER NOT NULL, + trigger_kind TEXT NOT NULL, + status TEXT NOT NULL, + parent_run_id TEXT NOT NULL DEFAULT '', + attempt INTEGER NOT NULL DEFAULT 1, + queued_at INTEGER NOT NULL DEFAULT 0, + started_at INTEGER NOT NULL DEFAULT 0, + finished_at INTEGER NOT NULL DEFAULT 0, + heartbeat_at INTEGER NOT NULL DEFAULT 0, + current_item INTEGER NOT NULL DEFAULT 0, + total_items INTEGER NOT NULL DEFAULT 0, + table_name TEXT NOT NULL DEFAULT '', + stage TEXT NOT NULL DEFAULT '', + rows_inserted INTEGER NOT NULL DEFAULT 0, + rows_updated INTEGER NOT NULL DEFAULT 0, + rows_deleted INTEGER NOT NULL DEFAULT 0, + rows_failed INTEGER NOT NULL DEFAULT 0, + message TEXT NOT NULL DEFAULT '', + resumable INTEGER NOT NULL DEFAULT 0, + definition_snapshot BLOB NOT NULL, + source_fingerprint TEXT NOT NULL DEFAULT '', + target_fingerprint TEXT NOT NULL DEFAULT '', + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + FOREIGN KEY(job_id) REFERENCES data_sync_jobs(id) ON DELETE CASCADE + )`, + `CREATE INDEX IF NOT EXISTS idx_data_sync_runs_job_created ON data_sync_runs(job_id, created_at DESC)`, + `CREATE INDEX IF NOT EXISTS idx_data_sync_runs_status ON data_sync_runs(status, updated_at DESC)`, + `CREATE TABLE IF NOT EXISTS data_sync_checkpoints ( + job_id TEXT PRIMARY KEY, + version INTEGER NOT NULL, + kind TEXT NOT NULL, + run_id TEXT NOT NULL, + definition_revision INTEGER NOT NULL, + table_name TEXT NOT NULL, + phase TEXT NOT NULL, + cursor_type TEXT NOT NULL, + cursor_json BLOB, + watermark_json BLOB, + batch_sequence INTEGER NOT NULL DEFAULT 0, + schema_hash TEXT NOT NULL DEFAULT '', + updated_at INTEGER NOT NULL, + FOREIGN KEY(job_id) REFERENCES data_sync_jobs(id) ON DELETE CASCADE, + FOREIGN KEY(run_id) REFERENCES data_sync_runs(id) ON DELETE CASCADE + )`, + `CREATE TABLE IF NOT EXISTS data_sync_error_rows ( + id TEXT PRIMARY KEY, + run_id TEXT NOT NULL, + job_id TEXT NOT NULL, + source_table TEXT NOT NULL DEFAULT '', + target_table TEXT NOT NULL DEFAULT '', + operation TEXT NOT NULL DEFAULT '', + source_key_json BLOB, + payload_json BLOB, + payload_policy TEXT NOT NULL DEFAULT 'keys_only', + payload_hash TEXT NOT NULL DEFAULT '', + payload_size INTEGER NOT NULL DEFAULT 0, + error_text TEXT NOT NULL, + error_code TEXT NOT NULL DEFAULT '', + error_class TEXT NOT NULL DEFAULT '', + attempts INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL, + retry_owner TEXT NOT NULL DEFAULT '', + retry_lease_expires_at INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + FOREIGN KEY(job_id) REFERENCES data_sync_jobs(id) ON DELETE CASCADE, + FOREIGN KEY(run_id) REFERENCES data_sync_runs(id) ON DELETE CASCADE + )`, + `CREATE INDEX IF NOT EXISTS idx_data_sync_error_rows_run ON data_sync_error_rows(run_id, status, created_at DESC)`, + `CREATE TABLE IF NOT EXISTS data_sync_run_events ( + run_id TEXT NOT NULL, + sequence INTEGER NOT NULL, + job_id TEXT NOT NULL, + event_type TEXT NOT NULL, + status TEXT NOT NULL DEFAULT '', + current_item INTEGER NOT NULL DEFAULT 0, + total_items INTEGER NOT NULL DEFAULT 0, + table_name TEXT NOT NULL DEFAULT '', + stage TEXT NOT NULL DEFAULT '', + message TEXT NOT NULL DEFAULT '', + payload_json BLOB, + created_at INTEGER NOT NULL, + PRIMARY KEY(run_id, sequence), + FOREIGN KEY(job_id) REFERENCES data_sync_jobs(id) ON DELETE CASCADE, + FOREIGN KEY(run_id) REFERENCES data_sync_runs(id) ON DELETE CASCADE + )`, + `CREATE INDEX IF NOT EXISTS idx_data_sync_run_events_job ON data_sync_run_events(job_id, created_at DESC)`, + `CREATE TABLE IF NOT EXISTS data_sync_scheduler_leases ( + name TEXT PRIMARY KEY, + owner_id TEXT NOT NULL, + expires_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + )`, + } + for _, statement := range statements { + if _, err := s.db.ExecContext(ctx, statement); err != nil { + return fmt.Errorf("initialize data sync job database: %w", err) + } + } + if version < 3 { + hasOwnerToken, err := sqliteTableHasColumn(ctx, s.db, "data_sync_runs", "owner_token") + if err != nil { + return fmt.Errorf("inspect data sync job run ownership migration: %w", err) + } + if !hasOwnerToken { + if _, err := s.db.ExecContext(ctx, `ALTER TABLE data_sync_runs ADD COLUMN owner_token TEXT NOT NULL DEFAULT ''`); err != nil { + return fmt.Errorf("migrate data sync job run ownership: %w", err) + } + } + } + if version < 4 { + hasRetryOwner, err := sqliteTableHasColumn(ctx, s.db, "data_sync_error_rows", "retry_owner") + if err != nil { + return fmt.Errorf("inspect data sync error row retry owner migration: %w", err) + } + if !hasRetryOwner { + if _, err := s.db.ExecContext(ctx, `ALTER TABLE data_sync_error_rows ADD COLUMN retry_owner TEXT NOT NULL DEFAULT ''`); err != nil { + return fmt.Errorf("migrate data sync error row retry owner: %w", err) + } + } + hasRetryLease, err := sqliteTableHasColumn(ctx, s.db, "data_sync_error_rows", "retry_lease_expires_at") + if err != nil { + return fmt.Errorf("inspect data sync error row retry lease migration: %w", err) + } + if !hasRetryLease { + if _, err := s.db.ExecContext(ctx, `ALTER TABLE data_sync_error_rows ADD COLUMN retry_lease_expires_at INTEGER NOT NULL DEFAULT 0`); err != nil { + return fmt.Errorf("migrate data sync error row retry lease: %w", err) + } + } + } + if _, err := s.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_data_sync_error_rows_retry ON data_sync_error_rows(status, retry_lease_expires_at)`); err != nil { + return fmt.Errorf("initialize data sync error row retry index: %w", err) + } + if _, err := s.db.ExecContext(ctx, fmt.Sprintf("PRAGMA user_version=%d", storeSchemaVersion)); err != nil { + return fmt.Errorf("record data sync job schema version: %w", err) + } + return nil +} + +func sqliteTableHasColumn(ctx context.Context, database *sql.DB, table, column string) (bool, error) { + rows, err := database.QueryContext(ctx, `PRAGMA table_info(`+table+`)`) + if err != nil { + return false, err + } + defer rows.Close() + for rows.Next() { + var cid, notNull, primaryKey int + var name, dataType string + var defaultValue any + if err := rows.Scan(&cid, &name, &dataType, ¬Null, &defaultValue, &primaryKey); err != nil { + return false, err + } + if name == column { + return true, nil + } + } + return false, rows.Err() +} + +func (s *Store) PutJob(ctx context.Context, input JobDefinition) (JobDefinition, error) { + return s.putJob(ctx, input, true) +} + +func (s *Store) putJob(ctx context.Context, input JobDefinition, cancelInactiveRuns bool) (JobDefinition, error) { + if err := s.ensureOpen(); err != nil { + return JobDefinition{}, err + } + definition := NormalizeDefinition(input) + if err := ValidatePersistableDefinition(definition); err != nil { + return JobDefinition{}, err + } + now := time.Now().UnixMilli() + if definition.Lifecycle == JobLifecycleArchived { + if definition.ArchivedAt == 0 { + definition.ArchivedAt = now + } + } else { + definition.ArchivedAt = 0 + } + if definition.Schedule.Kind == ScheduleInterval && definition.Schedule.AnchorAt <= 0 { + definition.Schedule.AnchorAt = now + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return JobDefinition{}, fmt.Errorf("begin data sync job update: %w", err) + } + defer func() { _ = tx.Rollback() }() + + if definition.ID == "" { + definition.ID = "sync-job-" + uuid.NewString() + definition.Revision = 1 + definition.CreatedAt = now + } else { + var revision, createdAt int64 + err := tx.QueryRowContext(ctx, `SELECT revision, created_at FROM data_sync_jobs WHERE id = ?`, definition.ID).Scan(&revision, &createdAt) + switch { + case errors.Is(err, sql.ErrNoRows): + if definition.Revision > 0 { + return JobDefinition{}, ErrNotFound + } + definition.Revision = 1 + definition.CreatedAt = now + case err != nil: + return JobDefinition{}, fmt.Errorf("read data sync job revision: %w", err) + default: + if definition.Revision != revision { + return JobDefinition{}, fmt.Errorf("%w: expected %d, got %d", ErrRevisionConflict, revision, definition.Revision) + } + definition.Revision = revision + 1 + definition.CreatedAt = createdAt + } + } + definition.UpdatedAt = now + definition.NextRunAt = NextRunAt(definition, time.UnixMilli(now)) + payload, err := json.Marshal(definition) + if err != nil { + return JobDefinition{}, fmt.Errorf("encode data sync job: %w", err) + } + _, err = tx.ExecContext(ctx, `INSERT INTO data_sync_jobs( + id, name, version, enabled, job_kind, incremental_mode, source_connection_id, target_connection_id, + definition_json, revision, created_at, updated_at, next_run_at, last_scheduled_at, archived_at + ) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + name=excluded.name, + version=excluded.version, + enabled=excluded.enabled, + job_kind=excluded.job_kind, + incremental_mode=excluded.incremental_mode, + source_connection_id=excluded.source_connection_id, + target_connection_id=excluded.target_connection_id, + definition_json=excluded.definition_json, + revision=excluded.revision, + updated_at=excluded.updated_at, + next_run_at=excluded.next_run_at, + archived_at=excluded.archived_at`, + definition.ID, definition.Name, definition.Version, boolInt(definition.Enabled), definition.Kind, definition.IncrementalMode, + definition.Source.ConnectionID, definition.Target.ConnectionID, payload, definition.Revision, + definition.CreatedAt, definition.UpdatedAt, definition.NextRunAt, definition.LastScheduledAt, definition.ArchivedAt, + ) + if err != nil { + return JobDefinition{}, fmt.Errorf("save data sync job: %w", err) + } + if err := tx.Commit(); err != nil { + return JobDefinition{}, fmt.Errorf("commit data sync job update: %w", err) + } + if cancelInactiveRuns && (definition.Lifecycle == JobLifecyclePaused || definition.Lifecycle == JobLifecycleArchived) { + if _, err := s.requestCancelRunsForJob(ctx, definition.ID, now, string(definition.Lifecycle)); err != nil { + return JobDefinition{}, err + } + } + return definition, nil +} + +func (s *Store) PauseJob(ctx context.Context, id string) (JobDefinition, error) { + definition, err := s.GetJob(ctx, strings.TrimSpace(id)) + if err != nil { + return JobDefinition{}, err + } + if definition.Lifecycle == JobLifecycleArchived || definition.ArchivedAt != 0 { + return JobDefinition{}, ErrNotFound + } + definition.Lifecycle = JobLifecyclePaused + definition.Enabled = false + paused, err := s.putJob(ctx, definition, false) + if err != nil { + return JobDefinition{}, err + } + if _, err := s.requestCancelRunsForJob(ctx, paused.ID, time.Now().UnixMilli(), "paused"); err != nil { + return JobDefinition{}, err + } + return paused, nil +} + +func (s *Store) GetJob(ctx context.Context, id string) (JobDefinition, error) { + if err := s.ensureOpen(); err != nil { + return JobDefinition{}, err + } + return scanJob(s.db.QueryRowContext(ctx, `SELECT definition_json, next_run_at, last_scheduled_at, archived_at FROM data_sync_jobs WHERE id = ?`, strings.TrimSpace(id))) +} + +func (s *Store) ListJobs(ctx context.Context) ([]JobDefinition, error) { + if err := s.ensureOpen(); err != nil { + return nil, err + } + rows, err := s.db.QueryContext(ctx, `SELECT definition_json, next_run_at, last_scheduled_at, archived_at FROM data_sync_jobs WHERE archived_at = 0 ORDER BY updated_at DESC, id`) + if err != nil { + return nil, fmt.Errorf("list data sync jobs: %w", err) + } + defer rows.Close() + return scanJobs(rows) +} + +func (s *Store) ListDueJobs(ctx context.Context, nowMillis int64) ([]JobDefinition, error) { + if err := s.ensureOpen(); err != nil { + return nil, err + } + rows, err := s.db.QueryContext(ctx, `SELECT definition_json, next_run_at, last_scheduled_at, archived_at FROM data_sync_jobs + WHERE enabled = 1 AND archived_at = 0 AND next_run_at > 0 AND next_run_at <= ? ORDER BY next_run_at, id`, nowMillis) + if err != nil { + return nil, fmt.Errorf("list due data sync jobs: %w", err) + } + defer rows.Close() + return scanJobs(rows) +} + +func (s *Store) AdvanceSchedule(ctx context.Context, id string, now time.Time) error { + definition, err := s.GetJob(ctx, id) + if err != nil { + return err + } + scheduledAt := definition.NextRunAt + base := now + if definition.Schedule.MisfirePolicy == "catch_up" && scheduledAt > 0 { + base = time.UnixMilli(scheduledAt) + } + next := NextRunAt(definition, base) + result, err := s.db.ExecContext(ctx, `UPDATE data_sync_jobs SET next_run_at = ?, last_scheduled_at = ? WHERE id = ?`, next, scheduledAt, definition.ID) + if err != nil { + return fmt.Errorf("advance data sync job schedule: %w", err) + } + return requireAffected(result) +} + +func (s *Store) AdvanceScheduleIfDue(ctx context.Context, id string, scheduledAt int64, now time.Time) (bool, error) { + definition, err := s.GetJob(ctx, id) + if err != nil { + return false, err + } + if scheduledAt <= 0 || definition.NextRunAt != scheduledAt { + return false, nil + } + base := now + if definition.Schedule.MisfirePolicy == "catch_up" { + base = time.UnixMilli(scheduledAt) + } + next := NextRunAt(definition, base) + result, err := s.db.ExecContext(ctx, `UPDATE data_sync_jobs SET next_run_at = ?, last_scheduled_at = ? WHERE id = ? AND next_run_at = ? AND enabled = 1 AND archived_at = 0`, + next, scheduledAt, definition.ID, scheduledAt) + if err != nil { + return false, fmt.Errorf("advance due data sync job schedule: %w", err) + } + affected, err := result.RowsAffected() + if err != nil { + return false, fmt.Errorf("read advanced data sync job schedule count: %w", err) + } + return affected > 0, nil +} + +func (s *Store) DelayScheduleIfDue(ctx context.Context, id string, scheduledAt, notBefore int64) (bool, error) { + if err := s.ensureOpen(); err != nil { + return false, err + } + if scheduledAt <= 0 || notBefore <= scheduledAt { + return false, nil + } + result, err := s.db.ExecContext(ctx, `UPDATE data_sync_jobs SET next_run_at = ? + WHERE id = ? AND next_run_at = ? AND enabled = 1 AND archived_at = 0 + AND json_extract(definition_json, '$.lifecycle') = ?`, notBefore, strings.TrimSpace(id), scheduledAt, + JobLifecycleEnabled) + if err != nil { + return false, fmt.Errorf("delay due data sync job schedule: %w", err) + } + affected, err := result.RowsAffected() + if err != nil { + return false, fmt.Errorf("read delayed data sync job schedule count: %w", err) + } + return affected > 0, nil +} + +func (s *Store) DeleteJob(ctx context.Context, id string) error { + _, err := s.archiveJobAndCancelRuns(ctx, id, time.Now().UnixMilli()) + return err +} + +type archivedRunTransition struct { + Run RunRecord +} + +func (s *Store) archiveJobAndCancelRuns(ctx context.Context, id string, nowMillis int64) ([]archivedRunTransition, error) { + if err := s.ensureOpen(); err != nil { + return nil, err + } + definition, err := s.GetJob(ctx, strings.TrimSpace(id)) + if err != nil { + return nil, err + } + if definition.ArchivedAt != 0 || definition.Lifecycle == JobLifecycleArchived { + return nil, ErrNotFound + } + definition.Lifecycle = JobLifecycleArchived + definition.Enabled = false + if _, err = s.putJob(ctx, definition, false); err != nil { + return nil, err + } + return s.requestCancelRunsForJob(ctx, definition.ID, nowMillis, "archived") +} + +func (s *Store) requestCancelRunsForJob(ctx context.Context, jobID string, nowMillis int64, reason string) ([]archivedRunTransition, error) { + if nowMillis <= 0 { + nowMillis = time.Now().UnixMilli() + } + reason = strings.TrimSpace(reason) + if reason == "" { + reason = "disabled" + } + canceledMessage := "canceled because task was " + reason + cancellingMessage := "cancellation requested because task was " + reason + rows, err := s.db.QueryContext(ctx, `UPDATE data_sync_runs SET + status = CASE WHEN status IN (?, ?) THEN ? ELSE ? END, + finished_at = CASE WHEN status IN (?, ?) THEN ? ELSE finished_at END, + owner_token = CASE WHEN status IN (?, ?) THEN '' ELSE owner_token END, + message = CASE + WHEN status IN (?, ?) THEN ? + WHEN message = '' THEN ? + ELSE message + END, + updated_at = ? + WHERE job_id = ? AND status IN (?, ?, ?, ?) + RETURNING `+runColumns, + RunStatusQueued, RunStatusPaused, RunStatusCanceled, RunStatusCancelling, + RunStatusQueued, RunStatusPaused, nowMillis, + RunStatusQueued, RunStatusPaused, + RunStatusQueued, RunStatusPaused, + canceledMessage, cancellingMessage, + nowMillis, strings.TrimSpace(jobID), RunStatusQueued, RunStatusRunning, RunStatusCancelling, RunStatusPaused) + if err != nil { + return nil, fmt.Errorf("cancel runs for inactive data sync job: %w", err) + } + defer rows.Close() + transitions := make([]archivedRunTransition, 0) + for rows.Next() { + run, scanErr := scanRun(rows) + if scanErr != nil { + return nil, scanErr + } + transitions = append(transitions, archivedRunTransition{Run: run}) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate inactive data sync run cancellations: %w", err) + } + return transitions, nil +} + +func (s *Store) CreateRun(ctx context.Context, run RunRecord) (RunRecord, error) { + return s.createRun(ctx, run, false) +} + +func (s *Store) CreateRunWithPolicy(ctx context.Context, run RunRecord, policy string) (RunRecord, error) { + forbidPending, err := forbidPendingForPolicy(policy) + if err != nil { + return RunRecord{}, err + } + return s.createRun(ctx, run, forbidPending) +} + +func forbidPendingForPolicy(policy string) (bool, error) { + policy = strings.TrimSpace(policy) + if policy == "" { + policy = "forbid" + } + switch policy { + case "queue": + return false, nil + case "forbid": + return true, nil + default: + return false, fmt.Errorf("unsupported data sync concurrency policy %q", policy) + } +} + +func (s *Store) createRun(ctx context.Context, run RunRecord, forbidPending bool) (RunRecord, error) { + if err := s.ensureOpen(); err != nil { + return RunRecord{}, err + } + run, err := prepareRunForInsert(run) + if err != nil { + return RunRecord{}, err + } + if err := insertRun(ctx, s.db, run, forbidPending); err != nil { + return RunRecord{}, err + } + return run, nil +} + +func (s *Store) CreateRunWithPolicyAndQueuedEvent(ctx context.Context, run RunRecord, policy string, eventCreatedAt int64) (RunRecord, RunEvent, error) { + if err := s.ensureOpen(); err != nil { + return RunRecord{}, RunEvent{}, err + } + forbidPending, err := forbidPendingForPolicy(policy) + if err != nil { + return RunRecord{}, RunEvent{}, err + } + run, err = prepareRunForInsert(run) + if err != nil { + return RunRecord{}, RunEvent{}, err + } + if run.Status != RunStatusQueued { + return RunRecord{}, RunEvent{}, errors.New("atomic data sync run creation requires queued status") + } + if eventCreatedAt <= 0 { + eventCreatedAt = time.Now().UnixMilli() + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return RunRecord{}, RunEvent{}, fmt.Errorf("begin atomic data sync run creation: %w", err) + } + defer func() { _ = tx.Rollback() }() + if err := insertRun(ctx, tx, run, forbidPending); err != nil { + return RunRecord{}, RunEvent{}, err + } + event := RunEvent{ + RunID: run.ID, JobID: run.JobID, Sequence: 1, Type: RunEventQueued, Status: RunStatusQueued, + Message: "queued", CreatedAt: eventCreatedAt, + } + if _, err := tx.ExecContext(ctx, `INSERT INTO data_sync_run_events( + run_id, sequence, job_id, event_type, status, current_item, total_items, table_name, stage, message, payload_json, created_at + ) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?)`, event.RunID, event.Sequence, event.JobID, event.Type, + event.Status, event.Current, event.Total, event.Table, event.Stage, event.Message, event.CreatedAt); err != nil { + return RunRecord{}, RunEvent{}, fmt.Errorf("create queued data sync run event: %w", err) + } + if err := tx.Commit(); err != nil { + return RunRecord{}, RunEvent{}, fmt.Errorf("commit atomic data sync run creation: %w", err) + } + return run, event, nil +} + +func prepareRunForInsert(run RunRecord) (RunRecord, error) { + now := time.Now().UnixMilli() + if strings.TrimSpace(run.ID) == "" { + run.ID = "sync-run-" + uuid.NewString() + } + if run.Status == "" { + run.Status = RunStatusQueued + } + if run.Trigger == "" { + run.Trigger = RunTriggerManual + } + if run.Attempt < 1 { + run.Attempt = 1 + } + if run.QueuedAt == 0 { + run.QueuedAt = now + } + if len(run.DefinitionSnapshot) == 0 { + return RunRecord{}, errors.New("data sync run definition snapshot is required") + } + run.CreatedAt = now + run.UpdatedAt = now + return run, nil +} + +type contextExecer interface { + ExecContext(context.Context, string, ...any) (sql.Result, error) +} + +func insertRun(ctx context.Context, executor contextExecer, run RunRecord, forbidPending bool) error { + insertPrefix := `INSERT INTO data_sync_runs( + id, job_id, owner_token, job_revision, trigger_kind, status, parent_run_id, attempt, queued_at, started_at, finished_at, heartbeat_at, + current_item, total_items, table_name, stage, rows_inserted, rows_updated, + rows_deleted, rows_failed, message, resumable, definition_snapshot, source_fingerprint, target_fingerprint, created_at, updated_at + ) ` + values := `?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?` + query := insertPrefix + `VALUES(` + values + `)` + args := []any{ + run.ID, run.JobID, run.OwnerToken, run.JobRevision, run.Trigger, run.Status, run.ParentRunID, run.Attempt, run.QueuedAt, run.StartedAt, run.FinishedAt, run.HeartbeatAt, + run.Current, run.Total, run.Table, run.Stage, run.RowsInserted, run.RowsUpdated, + run.RowsDeleted, run.RowsFailed, run.Message, boolInt(run.Resumable), []byte(run.DefinitionSnapshot), run.SourceFingerprint, run.TargetFingerprint, run.CreatedAt, run.UpdatedAt, + } + if forbidPending { + query = insertPrefix + `SELECT ` + values + ` WHERE NOT EXISTS ( + SELECT 1 FROM data_sync_runs WHERE job_id = ? AND status IN (?, ?, ?, ?) + )` + args = append(args, run.JobID, RunStatusQueued, RunStatusRunning, RunStatusCancelling, RunStatusPaused) + } + result, err := executor.ExecContext(ctx, query, args...) + if err != nil { + return fmt.Errorf("create data sync run: %w", err) + } + if forbidPending { + affected, affectedErr := result.RowsAffected() + if affectedErr != nil { + return fmt.Errorf("read created data sync run count: %w", affectedErr) + } + if affected == 0 { + return ErrRunAlreadyActive + } + } + return nil +} + +func (s *Store) UpdateRun(ctx context.Context, run RunRecord) error { + if err := s.ensureOpen(); err != nil { + return err + } + run.UpdatedAt = time.Now().UnixMilli() + result, err := s.db.ExecContext(ctx, `UPDATE data_sync_runs SET + status=?, started_at=?, finished_at=?, heartbeat_at=?, current_item=?, total_items=?, table_name=?, stage=?, + rows_inserted=?, rows_updated=?, rows_deleted=?, rows_failed=?, message=?, resumable=?, updated_at=? + WHERE id=? AND owner_token = ''`, run.Status, run.StartedAt, run.FinishedAt, run.HeartbeatAt, run.Current, run.Total, run.Table, run.Stage, + run.RowsInserted, run.RowsUpdated, run.RowsDeleted, run.RowsFailed, run.Message, boolInt(run.Resumable), + run.UpdatedAt, run.ID) + if err != nil { + return fmt.Errorf("update data sync run: %w", err) + } + return requireUnownedAffected(result) +} + +func (s *Store) GetRun(ctx context.Context, id string) (RunRecord, error) { + if err := s.ensureOpen(); err != nil { + return RunRecord{}, err + } + return scanRun(s.db.QueryRowContext(ctx, runSelect+` WHERE id = ?`, strings.TrimSpace(id))) +} + +func (s *Store) ListRuns(ctx context.Context, jobID string, limit int) ([]RunRecord, error) { + if err := s.ensureOpen(); err != nil { + return nil, err + } + if limit < 1 { + limit = 50 + } + if limit > 500 { + limit = 500 + } + query := runSelect + args := make([]any, 0, 2) + if strings.TrimSpace(jobID) != "" { + query += ` WHERE job_id = ?` + args = append(args, strings.TrimSpace(jobID)) + } + query += ` ORDER BY created_at DESC, id LIMIT ?` + args = append(args, limit) + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("list data sync runs: %w", err) + } + defer rows.Close() + result := make([]RunRecord, 0) + for rows.Next() { + run, err := scanRun(rows) + if err != nil { + return nil, err + } + result = append(result, run) + } + return result, rows.Err() +} + +func (s *Store) ListQueuedRuns(ctx context.Context, limit int) ([]RunRecord, error) { + if err := s.ensureOpen(); err != nil { + return nil, err + } + if limit < 1 { + limit = 100 + } + if limit > 1000 { + limit = 1000 + } + rows, err := s.db.QueryContext(ctx, runSelect+` WHERE status = ? AND EXISTS ( + SELECT 1 FROM data_sync_jobs AS job WHERE job.id = data_sync_runs.job_id AND job.archived_at = 0 AND ( + json_extract(job.definition_json, '$.lifecycle') = ? OR + (json_extract(job.definition_json, '$.lifecycle') = ? AND job.enabled = 1) + ) + ) ORDER BY queued_at, created_at, id LIMIT ?`, RunStatusQueued, JobLifecycleReady, JobLifecycleEnabled, limit) + if err != nil { + return nil, fmt.Errorf("list queued data sync runs: %w", err) + } + defer rows.Close() + result := make([]RunRecord, 0) + for rows.Next() { + run, scanErr := scanRun(rows) + if scanErr != nil { + return nil, scanErr + } + result = append(result, run) + } + return result, rows.Err() +} + +func (s *Store) ClaimRun(ctx context.Context, id string, nowMillis int64) (RunRecord, bool, error) { + if err := s.ensureOpen(); err != nil { + return RunRecord{}, false, err + } + if nowMillis <= 0 { + nowMillis = time.Now().UnixMilli() + } + ownerToken := uuid.NewString() + result, err := s.db.ExecContext(ctx, `UPDATE data_sync_runs AS candidate SET + status = ?, started_at = CASE WHEN started_at = 0 THEN ? ELSE started_at END, + heartbeat_at = ?, owner_token = ?, updated_at = ? + WHERE id = ? AND status = ? AND EXISTS ( + SELECT 1 FROM data_sync_jobs AS job + WHERE job.id = candidate.job_id AND job.archived_at = 0 AND ( + json_extract(job.definition_json, '$.lifecycle') = ? OR + (json_extract(job.definition_json, '$.lifecycle') = ? AND job.enabled = 1) + ) + ) AND NOT EXISTS ( + SELECT 1 FROM data_sync_runs AS active + WHERE active.job_id = candidate.job_id AND active.id <> candidate.id AND active.status IN (?, ?) + )`, RunStatusRunning, nowMillis, nowMillis, ownerToken, nowMillis, strings.TrimSpace(id), RunStatusQueued, + JobLifecycleReady, JobLifecycleEnabled, + RunStatusRunning, RunStatusCancelling) + if err != nil { + return RunRecord{}, false, fmt.Errorf("claim data sync run: %w", err) + } + affected, err := result.RowsAffected() + if err != nil { + return RunRecord{}, false, fmt.Errorf("read claimed data sync run count: %w", err) + } + if affected == 0 { + return RunRecord{}, false, nil + } + run, err := s.GetRun(ctx, id) + if err != nil { + return RunRecord{}, false, err + } + return run, true, nil +} + +func (s *Store) TouchRun(ctx context.Context, id string, nowMillis int64) error { + if err := s.ensureOpen(); err != nil { + return err + } + if nowMillis <= 0 { + nowMillis = time.Now().UnixMilli() + } + result, err := s.db.ExecContext(ctx, `UPDATE data_sync_runs SET heartbeat_at = ?, updated_at = ? WHERE id = ? AND owner_token = '' AND status IN (?, ?)`, + nowMillis, nowMillis, strings.TrimSpace(id), RunStatusRunning, RunStatusCancelling) + if err != nil { + return fmt.Errorf("heartbeat data sync run: %w", err) + } + return requireUnownedAffected(result) +} + +func (s *Store) TouchRunOwned(ctx context.Context, id, ownerToken string, nowMillis int64) error { + if err := s.ensureOpen(); err != nil { + return err + } + if strings.TrimSpace(ownerToken) == "" { + return ErrRunOwnershipLost + } + if nowMillis <= 0 { + nowMillis = time.Now().UnixMilli() + } + result, err := s.db.ExecContext(ctx, `UPDATE data_sync_runs SET heartbeat_at = ?, updated_at = ? + WHERE id = ? AND owner_token = ? AND status IN (?, ?)`, nowMillis, nowMillis, + strings.TrimSpace(id), ownerToken, RunStatusRunning, RunStatusCancelling) + if err != nil { + return fmt.Errorf("heartbeat owned data sync run: %w", err) + } + return requireOwnedAffected(result) +} + +func (s *Store) UpdateRunProgress(ctx context.Context, id string, progress RunProgress, nowMillis int64) (RunRecord, error) { + if err := s.ensureOpen(); err != nil { + return RunRecord{}, err + } + if progress.Current < 0 || progress.Total < 0 || (progress.Total > 0 && progress.Current > progress.Total) { + return RunRecord{}, errors.New("data sync run progress is outside its valid range") + } + if nowMillis <= 0 { + nowMillis = time.Now().UnixMilli() + } + result, err := s.db.ExecContext(ctx, `UPDATE data_sync_runs SET current_item = ?, total_items = ?, table_name = ?, stage = ?, + message = ?, heartbeat_at = ?, updated_at = ? WHERE id = ? AND owner_token = '' AND status = ?`, + progress.Current, progress.Total, strings.TrimSpace(progress.Table), strings.TrimSpace(progress.Stage), progress.Message, + nowMillis, nowMillis, strings.TrimSpace(id), RunStatusRunning) + if err != nil { + return RunRecord{}, fmt.Errorf("update data sync run progress: %w", err) + } + if err := requireUnownedAffected(result); err != nil { + return RunRecord{}, err + } + return s.GetRun(ctx, id) +} + +func (s *Store) UpdateRunProgressOwned(ctx context.Context, id, ownerToken string, progress RunProgress, nowMillis int64) (RunRecord, error) { + if err := s.ensureOpen(); err != nil { + return RunRecord{}, err + } + if strings.TrimSpace(ownerToken) == "" { + return RunRecord{}, ErrRunOwnershipLost + } + if progress.Current < 0 || progress.Total < 0 || (progress.Total > 0 && progress.Current > progress.Total) { + return RunRecord{}, errors.New("data sync run progress is outside its valid range") + } + if nowMillis <= 0 { + nowMillis = time.Now().UnixMilli() + } + result, err := s.db.ExecContext(ctx, `UPDATE data_sync_runs SET current_item = ?, total_items = ?, table_name = ?, stage = ?, + message = ?, heartbeat_at = ?, updated_at = ? WHERE id = ? AND owner_token = ? AND status = ?`, + progress.Current, progress.Total, strings.TrimSpace(progress.Table), strings.TrimSpace(progress.Stage), progress.Message, + nowMillis, nowMillis, strings.TrimSpace(id), ownerToken, RunStatusRunning) + if err != nil { + return RunRecord{}, fmt.Errorf("update owned data sync run progress: %w", err) + } + if err := requireOwnedAffected(result); err != nil { + return RunRecord{}, err + } + return s.GetRun(ctx, id) +} + +func (s *Store) CompleteRun(ctx context.Context, id string, status RunStatus, outcome ExecutionOutcome, message string, nowMillis int64) (RunRecord, error) { + if err := s.ensureOpen(); err != nil { + return RunRecord{}, err + } + switch status { + case RunStatusSucceeded, RunStatusPartial, RunStatusFailed, RunStatusCanceled, RunStatusInterrupted: + default: + return RunRecord{}, fmt.Errorf("unsupported terminal data sync run status %q", status) + } + if nowMillis <= 0 { + nowMillis = time.Now().UnixMilli() + } + if message == "" { + message = outcome.Message + } + resumable := outcome.Resumable || status == RunStatusInterrupted + result, err := s.db.ExecContext(ctx, `UPDATE data_sync_runs SET status = ?, finished_at = ?, heartbeat_at = ?, owner_token = '', + rows_inserted = ?, rows_updated = ?, rows_deleted = ?, rows_failed = ?, message = ?, resumable = ?, updated_at = ? + WHERE id = ? AND owner_token = '' AND status IN (?, ?)`, status, nowMillis, nowMillis, outcome.RowsInserted, outcome.RowsUpdated, + outcome.RowsDeleted, outcome.RowsFailed, message, boolInt(resumable), nowMillis, strings.TrimSpace(id), + RunStatusRunning, RunStatusCancelling) + if err != nil { + return RunRecord{}, fmt.Errorf("complete data sync run: %w", err) + } + if err := requireUnownedAffected(result); err != nil { + return RunRecord{}, err + } + return s.GetRun(ctx, id) +} + +func (s *Store) CompleteRunOwned(ctx context.Context, id, ownerToken string, status RunStatus, outcome ExecutionOutcome, message string, nowMillis int64) (RunRecord, error) { + if err := s.ensureOpen(); err != nil { + return RunRecord{}, err + } + if strings.TrimSpace(ownerToken) == "" { + return RunRecord{}, ErrRunOwnershipLost + } + switch status { + case RunStatusSucceeded, RunStatusPartial, RunStatusFailed, RunStatusCanceled, RunStatusInterrupted: + default: + return RunRecord{}, fmt.Errorf("unsupported terminal data sync run status %q", status) + } + if nowMillis <= 0 { + nowMillis = time.Now().UnixMilli() + } + if message == "" { + message = outcome.Message + } + resumable := outcome.Resumable || status == RunStatusInterrupted + result, err := s.db.ExecContext(ctx, `UPDATE data_sync_runs SET status = ?, finished_at = ?, heartbeat_at = ?, owner_token = '', + rows_inserted = ?, rows_updated = ?, rows_deleted = ?, rows_failed = ?, message = ?, resumable = ?, updated_at = ? + WHERE id = ? AND owner_token = ? AND status IN (?, ?)`, status, nowMillis, nowMillis, outcome.RowsInserted, outcome.RowsUpdated, + outcome.RowsDeleted, outcome.RowsFailed, message, boolInt(resumable), nowMillis, strings.TrimSpace(id), ownerToken, + RunStatusRunning, RunStatusCancelling) + if err != nil { + return RunRecord{}, fmt.Errorf("complete owned data sync run: %w", err) + } + if err := requireOwnedAffected(result); err != nil { + return RunRecord{}, err + } + return s.GetRun(ctx, id) +} + +func (s *Store) RequestCancelRun(ctx context.Context, id string, nowMillis int64) (RunRecord, error) { + if err := s.ensureOpen(); err != nil { + return RunRecord{}, err + } + if nowMillis <= 0 { + nowMillis = time.Now().UnixMilli() + } + result, err := s.db.ExecContext(ctx, `UPDATE data_sync_runs SET + status = CASE WHEN status IN (?, ?) THEN ? ELSE ? END, + finished_at = CASE WHEN status IN (?, ?) THEN ? ELSE finished_at END, + message = CASE WHEN status IN (?, ?) THEN 'canceled before execution' ELSE message END, + owner_token = CASE WHEN status IN (?, ?) THEN '' ELSE owner_token END, + updated_at = ? WHERE id = ? AND status IN (?, ?, ?, ?)`, + RunStatusQueued, RunStatusPaused, RunStatusCanceled, RunStatusCancelling, + RunStatusQueued, RunStatusPaused, nowMillis, + RunStatusQueued, RunStatusPaused, + RunStatusQueued, RunStatusPaused, + nowMillis, strings.TrimSpace(id), RunStatusQueued, RunStatusRunning, RunStatusCancelling, RunStatusPaused) + if err != nil { + return RunRecord{}, fmt.Errorf("request data sync run cancellation: %w", err) + } + affected, err := result.RowsAffected() + if err != nil { + return RunRecord{}, fmt.Errorf("read canceled data sync run count: %w", err) + } + if affected == 0 { + if _, getErr := s.GetRun(ctx, id); getErr != nil { + return RunRecord{}, getErr + } + return RunRecord{}, ErrRunNotCancelable + } + return s.GetRun(ctx, id) +} + +func (s *Store) InterruptStaleRuns(ctx context.Context, cutoffMillis, nowMillis int64) ([]RunRecord, error) { + if err := s.ensureOpen(); err != nil { + return nil, err + } + if nowMillis <= 0 { + nowMillis = time.Now().UnixMilli() + } + if cutoffMillis <= 0 { + cutoffMillis = nowMillis + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return nil, fmt.Errorf("begin stale data sync run recovery: %w", err) + } + defer func() { _ = tx.Rollback() }() + rows, err := tx.QueryContext(ctx, runSelect+` WHERE status IN (?, ?) AND (heartbeat_at = 0 OR heartbeat_at <= ?) ORDER BY updated_at, id`, + RunStatusRunning, RunStatusCancelling, cutoffMillis) + if err != nil { + return nil, fmt.Errorf("list stale data sync runs: %w", err) + } + candidates := make([]RunRecord, 0) + for rows.Next() { + run, scanErr := scanRun(rows) + if scanErr != nil { + _ = rows.Close() + return nil, scanErr + } + candidates = append(candidates, run) + } + if err := rows.Close(); err != nil { + return nil, fmt.Errorf("close stale data sync run rows: %w", err) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate stale data sync runs: %w", err) + } + recovered := make([]RunRecord, 0, len(candidates)) + for _, candidate := range candidates { + status := RunStatusInterrupted + message := "interrupted after manager restart" + resumable := 1 + if candidate.Status == RunStatusCancelling { + status = RunStatusCanceled + message = "canceled after manager restart" + resumable = 0 + } + result, updateErr := tx.ExecContext(ctx, `UPDATE data_sync_runs SET status = ?, finished_at = ?, heartbeat_at = ?, owner_token = '', + message = CASE WHEN message = '' THEN ? ELSE message END, + resumable = ?, updated_at = ? WHERE id = ? AND status = ? AND owner_token = ? AND (heartbeat_at = 0 OR heartbeat_at <= ?)`, + status, nowMillis, nowMillis, message, resumable, nowMillis, candidate.ID, candidate.Status, candidate.OwnerToken, cutoffMillis) + if updateErr != nil { + return nil, fmt.Errorf("recover stale data sync run %s: %w", candidate.ID, updateErr) + } + affected, affectedErr := result.RowsAffected() + if affectedErr != nil { + return nil, fmt.Errorf("read recovered data sync run count: %w", affectedErr) + } + if affected == 0 { + continue + } + run, getErr := scanRun(tx.QueryRowContext(ctx, runSelect+` WHERE id = ?`, candidate.ID)) + if getErr != nil { + return nil, getErr + } + recovered = append(recovered, run) + } + if err := tx.Commit(); err != nil { + return nil, fmt.Errorf("commit stale data sync run recovery: %w", err) + } + return recovered, nil +} + +func (s *Store) PutCheckpoint(ctx context.Context, checkpoint Checkpoint) (Checkpoint, error) { + if err := s.ensureOpen(); err != nil { + return Checkpoint{}, err + } + checkpoint.JobID = strings.TrimSpace(checkpoint.JobID) + checkpoint.RunID = strings.TrimSpace(checkpoint.RunID) + checkpoint.Table = strings.TrimSpace(checkpoint.Table) + checkpoint.Phase = strings.TrimSpace(checkpoint.Phase) + if checkpoint.Version == 0 { + checkpoint.Version = 1 + } + checkpoint.Kind = strings.TrimSpace(checkpoint.Kind) + checkpoint.CursorType = strings.TrimSpace(checkpoint.CursorType) + if checkpoint.JobID == "" || checkpoint.RunID == "" || checkpoint.Table == "" || checkpoint.Phase == "" || checkpoint.Kind == "" || checkpoint.CursorType == "" { + return Checkpoint{}, errors.New("checkpoint requires jobId, runId, table, and phase") + } + if !validJSONOrEmpty(checkpoint.Cursor) || !validJSONOrEmpty(checkpoint.Watermark) { + return Checkpoint{}, errors.New("checkpoint cursor and watermark must be valid JSON") + } + checkpoint.UpdatedAt = time.Now().UnixMilli() + result, err := s.db.ExecContext(ctx, `INSERT INTO data_sync_checkpoints(job_id, version, kind, run_id, definition_revision, table_name, phase, cursor_type, cursor_json, watermark_json, batch_sequence, schema_hash, updated_at) + SELECT ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? FROM data_sync_runs AS owner + WHERE owner.id = ? AND owner.job_id = ? AND owner.owner_token = '' + ON CONFLICT(job_id) DO UPDATE SET version=excluded.version, kind=excluded.kind, run_id=excluded.run_id, + definition_revision=excluded.definition_revision, table_name=excluded.table_name, phase=excluded.phase, + cursor_type=excluded.cursor_type, cursor_json=excluded.cursor_json, watermark_json=excluded.watermark_json, + batch_sequence=excluded.batch_sequence, schema_hash=excluded.schema_hash, updated_at=excluded.updated_at`, + checkpoint.JobID, checkpoint.Version, checkpoint.Kind, checkpoint.RunID, checkpoint.DefinitionRevision, + checkpoint.Table, checkpoint.Phase, checkpoint.CursorType, nullableBytes(checkpoint.Cursor), + nullableBytes(checkpoint.Watermark), checkpoint.BatchSequence, checkpoint.SchemaHash, checkpoint.UpdatedAt, + checkpoint.RunID, checkpoint.JobID) + if err != nil { + return Checkpoint{}, fmt.Errorf("save data sync checkpoint: %w", err) + } + if err := requireUnownedAffected(result); err != nil { + return Checkpoint{}, err + } + return checkpoint, nil +} + +func (s *Store) PutCheckpointOwned(ctx context.Context, checkpoint Checkpoint, ownerToken string) (Checkpoint, error) { + if err := s.ensureOpen(); err != nil { + return Checkpoint{}, err + } + if strings.TrimSpace(ownerToken) == "" { + return Checkpoint{}, ErrRunOwnershipLost + } + checkpoint.JobID = strings.TrimSpace(checkpoint.JobID) + checkpoint.RunID = strings.TrimSpace(checkpoint.RunID) + checkpoint.Table = strings.TrimSpace(checkpoint.Table) + checkpoint.Phase = strings.TrimSpace(checkpoint.Phase) + if checkpoint.Version == 0 { + checkpoint.Version = 1 + } + checkpoint.Kind = strings.TrimSpace(checkpoint.Kind) + checkpoint.CursorType = strings.TrimSpace(checkpoint.CursorType) + if checkpoint.JobID == "" || checkpoint.RunID == "" || checkpoint.Table == "" || checkpoint.Phase == "" || checkpoint.Kind == "" || checkpoint.CursorType == "" { + return Checkpoint{}, errors.New("checkpoint requires jobId, runId, table, and phase") + } + if !validJSONOrEmpty(checkpoint.Cursor) || !validJSONOrEmpty(checkpoint.Watermark) { + return Checkpoint{}, errors.New("checkpoint cursor and watermark must be valid JSON") + } + checkpoint.UpdatedAt = time.Now().UnixMilli() + result, err := s.db.ExecContext(ctx, `INSERT INTO data_sync_checkpoints(job_id, version, kind, run_id, definition_revision, table_name, phase, cursor_type, cursor_json, watermark_json, batch_sequence, schema_hash, updated_at) + SELECT ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? FROM data_sync_runs AS owner + WHERE owner.id = ? AND owner.job_id = ? AND owner.owner_token = ? AND owner.status = ? + ON CONFLICT(job_id) DO UPDATE SET version=excluded.version, kind=excluded.kind, run_id=excluded.run_id, + definition_revision=excluded.definition_revision, table_name=excluded.table_name, phase=excluded.phase, + cursor_type=excluded.cursor_type, cursor_json=excluded.cursor_json, watermark_json=excluded.watermark_json, + batch_sequence=excluded.batch_sequence, schema_hash=excluded.schema_hash, updated_at=excluded.updated_at`, + checkpoint.JobID, checkpoint.Version, checkpoint.Kind, checkpoint.RunID, checkpoint.DefinitionRevision, + checkpoint.Table, checkpoint.Phase, checkpoint.CursorType, nullableBytes(checkpoint.Cursor), + nullableBytes(checkpoint.Watermark), checkpoint.BatchSequence, checkpoint.SchemaHash, checkpoint.UpdatedAt, + checkpoint.RunID, checkpoint.JobID, ownerToken, RunStatusRunning) + if err != nil { + return Checkpoint{}, fmt.Errorf("save owned data sync checkpoint: %w", err) + } + if err := requireOwnedAffected(result); err != nil { + return Checkpoint{}, err + } + return checkpoint, nil +} + +func (s *Store) GetCheckpoint(ctx context.Context, jobID string) (Checkpoint, error) { + if err := s.ensureOpen(); err != nil { + return Checkpoint{}, err + } + var checkpoint Checkpoint + var cursor, watermark []byte + err := s.db.QueryRowContext(ctx, `SELECT job_id, version, kind, run_id, definition_revision, table_name, phase, cursor_type, cursor_json, watermark_json, batch_sequence, schema_hash, updated_at + FROM data_sync_checkpoints WHERE job_id = ?`, strings.TrimSpace(jobID)).Scan(&checkpoint.JobID, &checkpoint.Version, + &checkpoint.Kind, &checkpoint.RunID, &checkpoint.DefinitionRevision, &checkpoint.Table, &checkpoint.Phase, + &checkpoint.CursorType, &cursor, &watermark, &checkpoint.BatchSequence, &checkpoint.SchemaHash, &checkpoint.UpdatedAt) + if errors.Is(err, sql.ErrNoRows) { + return Checkpoint{}, ErrNotFound + } + if err != nil { + return Checkpoint{}, fmt.Errorf("read data sync checkpoint: %w", err) + } + checkpoint.Cursor = cloneRaw(cursor) + checkpoint.Watermark = cloneRaw(watermark) + return checkpoint, nil +} + +func (s *Store) DeleteCheckpoint(ctx context.Context, jobID string) error { + if err := s.ensureOpen(); err != nil { + return err + } + jobID = strings.TrimSpace(jobID) + result, err := s.db.ExecContext(ctx, `DELETE FROM data_sync_checkpoints WHERE job_id = ? AND NOT EXISTS ( + SELECT 1 FROM data_sync_runs WHERE job_id = ? AND owner_token <> '' AND status IN (?, ?) + )`, jobID, jobID, RunStatusRunning, RunStatusCancelling) + if err != nil { + return fmt.Errorf("delete data sync checkpoint: %w", err) + } + affected, err := result.RowsAffected() + if err != nil { + return err + } + if affected == 0 { + var owned int + if err := s.db.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM data_sync_runs + WHERE job_id = ? AND owner_token <> '' AND status IN (?, ?))`, jobID, RunStatusRunning, RunStatusCancelling).Scan(&owned); err != nil { + return fmt.Errorf("verify data sync run ownership after checkpoint delete: %w", err) + } + if owned != 0 { + return ErrRunOwnershipLost + } + } + return nil +} + +func (s *Store) DeleteCheckpointOwned(ctx context.Context, jobID, runID, ownerToken string) error { + if err := s.ensureOpen(); err != nil { + return err + } + if strings.TrimSpace(ownerToken) == "" { + return ErrRunOwnershipLost + } + jobID = strings.TrimSpace(jobID) + runID = strings.TrimSpace(runID) + result, err := s.db.ExecContext(ctx, `DELETE FROM data_sync_checkpoints WHERE job_id = ? AND EXISTS ( + SELECT 1 FROM data_sync_runs AS owner WHERE owner.id = ? AND owner.job_id = ? + AND owner.owner_token = ? AND owner.status = ? + )`, jobID, runID, jobID, ownerToken, RunStatusRunning) + if err != nil { + return fmt.Errorf("delete owned data sync checkpoint: %w", err) + } + affected, err := result.RowsAffected() + if err != nil { + return err + } + if affected > 0 { + return nil + } + var owned int + if err := s.db.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM data_sync_runs + WHERE id = ? AND job_id = ? AND owner_token = ? AND status = ?)`, runID, jobID, ownerToken, RunStatusRunning).Scan(&owned); err != nil { + return fmt.Errorf("verify data sync run ownership after checkpoint delete: %w", err) + } + if owned == 0 { + return ErrRunOwnershipLost + } + return nil +} + +func (s *Store) ResetCheckpoint(ctx context.Context, jobID string) error { + if err := s.ensureOpen(); err != nil { + return err + } + jobID = strings.TrimSpace(jobID) + result, err := s.db.ExecContext(ctx, `DELETE FROM data_sync_checkpoints + WHERE job_id = ? AND NOT EXISTS ( + SELECT 1 FROM data_sync_runs + WHERE job_id = ? AND status IN (?, ?, ?) + )`, jobID, jobID, RunStatusQueued, RunStatusRunning, RunStatusCancelling) + if err != nil { + return fmt.Errorf("reset data sync checkpoint: %w", err) + } + affected, err := result.RowsAffected() + if err != nil { + return fmt.Errorf("read reset data sync checkpoint count: %w", err) + } + if affected > 0 { + return nil + } + var active int + if err := s.db.QueryRowContext(ctx, `SELECT EXISTS( + SELECT 1 FROM data_sync_runs WHERE job_id = ? AND status IN (?, ?, ?) + )`, jobID, RunStatusQueued, RunStatusRunning, RunStatusCancelling).Scan(&active); err != nil { + return fmt.Errorf("check active data sync run before checkpoint reset: %w", err) + } + if active != 0 { + return ErrRunAlreadyActive + } + return ErrNotFound +} + +func (s *Store) AppendErrorRow(ctx context.Context, row ErrorRow) (ErrorRow, error) { + if err := s.ensureOpen(); err != nil { + return ErrorRow{}, err + } + if strings.TrimSpace(row.RunID) == "" || strings.TrimSpace(row.JobID) == "" || strings.TrimSpace(row.Error) == "" { + return ErrorRow{}, errors.New("error row requires runId, jobId, and error") + } + if !validJSONOrEmpty(row.SourceKey) || !validJSONOrEmpty(row.Payload) { + return ErrorRow{}, errors.New("error row source key and payload must be valid JSON") + } + const ( + maxErrorRowSourceKeyBytes = 64 << 10 + maxErrorRowPayloadBytes = 1 << 20 + ) + if len(row.SourceKey) > maxErrorRowSourceKeyBytes { + return ErrorRow{}, fmt.Errorf("error row source key exceeds %d bytes", maxErrorRowSourceKeyBytes) + } + if len(row.Payload) > maxErrorRowPayloadBytes { + return ErrorRow{}, fmt.Errorf("error row payload exceeds %d bytes", maxErrorRowPayloadBytes) + } + if row.ID == "" { + row.ID = "sync-error-" + uuid.NewString() + } + if row.Status == "" { + row.Status = ErrorRowPending + } + if row.Status != ErrorRowPending { + return ErrorRow{}, fmt.Errorf("unsupported initial error row status %q", row.Status) + } + if row.PayloadPolicy == "" { + row.PayloadPolicy = "keys_only" + } + if row.PayloadPolicy != "none" && row.PayloadPolicy != "keys_only" && row.PayloadPolicy != "full" { + return ErrorRow{}, fmt.Errorf("unsupported error row payload policy %q", row.PayloadPolicy) + } + if row.PayloadPolicy != "full" { + row.Payload = nil + } + now := time.Now().UnixMilli() + row.CreatedAt = now + row.UpdatedAt = now + _, err := s.db.ExecContext(ctx, `INSERT INTO data_sync_error_rows( + id, run_id, job_id, source_table, target_table, operation, source_key_json, payload_json, + payload_policy, payload_hash, payload_size, error_text, error_code, error_class, attempts, status, created_at, updated_at + ) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, row.ID, row.RunID, row.JobID, + row.SourceTable, row.TargetTable, row.Operation, nullableBytes(row.SourceKey), nullableBytes(row.Payload), + row.PayloadPolicy, row.PayloadHash, row.PayloadSize, row.Error, row.ErrorCode, row.ErrorClass, + row.Attempts, row.Status, row.CreatedAt, row.UpdatedAt) + if err != nil { + return ErrorRow{}, fmt.Errorf("append data sync error row: %w", err) + } + return row, nil +} + +func (s *Store) ListErrorRows(ctx context.Context, runID string, status ErrorRowStatus, limit int) ([]ErrorRow, error) { + if err := s.ensureOpen(); err != nil { + return nil, err + } + if limit < 1 { + limit = 100 + } + if limit > 1000 { + limit = 1000 + } + query := `SELECT ` + errorRowColumns + ` FROM data_sync_error_rows WHERE run_id = ?` + args := []any{strings.TrimSpace(runID)} + if status != "" { + query += ` AND status = ?` + args = append(args, status) + } + query += ` ORDER BY created_at DESC, id LIMIT ?` + args = append(args, limit) + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("list data sync error rows: %w", err) + } + defer rows.Close() + result := make([]ErrorRow, 0) + for rows.Next() { + row, err := scanErrorRow(rows) + if err != nil { + return nil, err + } + result = append(result, row) + } + return result, rows.Err() +} + +func (s *Store) GetErrorRow(ctx context.Context, id string) (ErrorRow, error) { + if err := s.ensureOpen(); err != nil { + return ErrorRow{}, err + } + return scanErrorRow(s.db.QueryRowContext(ctx, `SELECT `+errorRowColumns+` FROM data_sync_error_rows WHERE id = ?`, strings.TrimSpace(id))) +} + +func (s *Store) ClaimErrorRowRetry(ctx context.Context, id string, nowMillis int64, leaseTTL time.Duration) (ErrorRow, error) { + if err := s.ensureOpen(); err != nil { + return ErrorRow{}, err + } + id = strings.TrimSpace(id) + if id == "" { + return ErrorRow{}, errors.New("data sync error row id is required") + } + leaseMillis := leaseTTL.Milliseconds() + if leaseMillis <= 0 { + return ErrorRow{}, errors.New("data sync error row retry lease must be positive") + } + if nowMillis <= 0 { + nowMillis = time.Now().UnixMilli() + } + owner := uuid.NewString() + leaseExpiresAt := nowMillis + leaseMillis + result, err := s.db.ExecContext(ctx, `UPDATE data_sync_error_rows SET + status=?, retry_owner=?, retry_lease_expires_at=?, + attempts=attempts+CASE WHEN status=? THEN 1 ELSE 0 END, updated_at=? + WHERE id=? AND (status=? OR (status=? AND retry_lease_expires_at <= ?))`, + ErrorRowRetrying, owner, leaseExpiresAt, ErrorRowRetrying, nowMillis, + id, ErrorRowPending, ErrorRowRetrying, nowMillis) + if err != nil { + return ErrorRow{}, fmt.Errorf("claim data sync error row retry: %w", err) + } + affected, err := result.RowsAffected() + if err != nil { + return ErrorRow{}, fmt.Errorf("read claimed data sync error row count: %w", err) + } + if affected == 0 { + if _, err := s.GetErrorRow(ctx, id); err != nil { + return ErrorRow{}, err + } + return ErrorRow{}, ErrErrorRowStateConflict + } + claimed, err := s.GetErrorRow(ctx, id) + if err != nil { + return ErrorRow{}, err + } + if claimed.Status != ErrorRowRetrying || claimed.RetryOwner != owner { + return ErrorRow{}, ErrErrorRowRetryOwnershipLost + } + return claimed, nil +} + +func (s *Store) RenewErrorRowRetry(ctx context.Context, id, owner string, nowMillis int64, leaseTTL time.Duration) error { + if err := s.ensureOpen(); err != nil { + return err + } + id = strings.TrimSpace(id) + owner = strings.TrimSpace(owner) + if id == "" || owner == "" { + return errors.New("data sync error row retry renewal requires id and owner") + } + leaseMillis := leaseTTL.Milliseconds() + if leaseMillis <= 0 { + return errors.New("data sync error row retry lease must be positive") + } + if nowMillis <= 0 { + nowMillis = time.Now().UnixMilli() + } + result, err := s.db.ExecContext(ctx, `UPDATE data_sync_error_rows SET retry_lease_expires_at=?, updated_at=? + WHERE id=? AND status=? AND retry_owner=?`, nowMillis+leaseMillis, nowMillis, + id, ErrorRowRetrying, owner) + if err != nil { + return fmt.Errorf("renew data sync error row retry: %w", err) + } + return s.requireErrorRowRetryOwnerAffected(ctx, id, result) +} + +func (s *Store) ResolveErrorRowRetry(ctx context.Context, id, owner string, nowMillis int64) error { + return s.finishErrorRowRetry(ctx, id, owner, ErrorRowResolved, nowMillis) +} + +func (s *Store) FailErrorRowRetry(ctx context.Context, id, owner string, nowMillis int64) error { + return s.finishErrorRowRetry(ctx, id, owner, ErrorRowPending, nowMillis) +} + +func (s *Store) finishErrorRowRetry(ctx context.Context, id, owner string, status ErrorRowStatus, nowMillis int64) error { + if err := s.ensureOpen(); err != nil { + return err + } + id = strings.TrimSpace(id) + owner = strings.TrimSpace(owner) + if id == "" || owner == "" { + return errors.New("data sync error row retry completion requires id and owner") + } + if status != ErrorRowPending && status != ErrorRowResolved { + return fmt.Errorf("unsupported data sync error row retry completion status %q", status) + } + if nowMillis <= 0 { + nowMillis = time.Now().UnixMilli() + } + result, err := s.db.ExecContext(ctx, `UPDATE data_sync_error_rows SET status=?, attempts=attempts+1, + retry_owner='', retry_lease_expires_at=0, updated_at=? WHERE id=? AND status=? AND retry_owner=?`, + status, nowMillis, id, ErrorRowRetrying, owner) + if err != nil { + return fmt.Errorf("complete data sync error row retry: %w", err) + } + return s.requireErrorRowRetryOwnerAffected(ctx, id, result) +} + +func (s *Store) RecoverExpiredErrorRowRetries(ctx context.Context, nowMillis int64) (int64, error) { + if err := s.ensureOpen(); err != nil { + return 0, err + } + if nowMillis <= 0 { + nowMillis = time.Now().UnixMilli() + } + result, err := s.db.ExecContext(ctx, `UPDATE data_sync_error_rows SET status=?, attempts=attempts+1, + retry_owner='', retry_lease_expires_at=0, updated_at=? WHERE status=? AND retry_lease_expires_at <= ?`, + ErrorRowPending, nowMillis, ErrorRowRetrying, nowMillis) + if err != nil { + return 0, fmt.Errorf("recover expired data sync error row retries: %w", err) + } + affected, err := result.RowsAffected() + if err != nil { + return 0, fmt.Errorf("read recovered data sync error row retry count: %w", err) + } + return affected, nil +} + +func (s *Store) requireErrorRowRetryOwnerAffected(ctx context.Context, id string, result sql.Result) error { + affected, err := result.RowsAffected() + if err != nil { + return fmt.Errorf("read updated data sync error row retry count: %w", err) + } + if affected > 0 { + return nil + } + if _, err := s.GetErrorRow(ctx, id); err != nil { + return err + } + return ErrErrorRowRetryOwnershipLost +} + +func (s *Store) UpdateErrorRowStatus(ctx context.Context, id string, status ErrorRowStatus, incrementAttempts bool) error { + if err := s.ensureOpen(); err != nil { + return err + } + if status != ErrorRowResolved && status != ErrorRowDiscarded { + return fmt.Errorf("unsupported error row status %q", status) + } + attemptDelta := 0 + if incrementAttempts { + attemptDelta = 1 + } + result, err := s.db.ExecContext(ctx, `UPDATE data_sync_error_rows SET status=?, attempts=attempts+?, updated_at=? WHERE id=? AND status=?`, + status, attemptDelta, time.Now().UnixMilli(), strings.TrimSpace(id), ErrorRowPending) + if err != nil { + return fmt.Errorf("update data sync error row: %w", err) + } + affected, err := result.RowsAffected() + if err != nil { + return fmt.Errorf("read updated data sync error row count: %w", err) + } + if affected > 0 { + return nil + } + if _, err := s.GetErrorRow(ctx, id); err != nil { + return err + } + return ErrErrorRowStateConflict +} + +// IncrementErrorRowAttempts records a failed replay while keeping the row +// pending. The CAS prevents a concurrent discard/resolve from being undone. +func (s *Store) IncrementErrorRowAttempts(ctx context.Context, id string) error { + if err := s.ensureOpen(); err != nil { + return err + } + result, err := s.db.ExecContext(ctx, `UPDATE data_sync_error_rows SET attempts=attempts+1, updated_at=? WHERE id=? AND status=?`, + time.Now().UnixMilli(), strings.TrimSpace(id), ErrorRowPending) + if err != nil { + return fmt.Errorf("increment data sync error row attempts: %w", err) + } + affected, err := result.RowsAffected() + if err != nil { + return fmt.Errorf("read incremented data sync error row count: %w", err) + } + if affected > 0 { + return nil + } + if _, err := s.GetErrorRow(ctx, id); err != nil { + return err + } + return ErrErrorRowStateConflict +} + +func (s *Store) AcquireSchedulerLease(ctx context.Context, name, owner string, now time.Time, ttl time.Duration) (bool, error) { + if err := s.ensureOpen(); err != nil { + return false, err + } + name = strings.TrimSpace(name) + owner = strings.TrimSpace(owner) + if name == "" || owner == "" { + return false, errors.New("scheduler lease requires name and owner") + } + if ttl <= 0 { + return false, errors.New("scheduler lease ttl must be positive") + } + nowMillis := now.UnixMilli() + expiresAt := now.Add(ttl).UnixMilli() + result, err := s.db.ExecContext(ctx, `INSERT INTO data_sync_scheduler_leases(name, owner_id, expires_at, updated_at) + VALUES(?, ?, ?, ?) + ON CONFLICT(name) DO UPDATE SET owner_id = excluded.owner_id, expires_at = excluded.expires_at, updated_at = excluded.updated_at + WHERE data_sync_scheduler_leases.owner_id = excluded.owner_id OR data_sync_scheduler_leases.expires_at <= ?`, + name, owner, expiresAt, nowMillis, nowMillis) + if err != nil { + return false, fmt.Errorf("acquire data sync scheduler lease: %w", err) + } + affected, err := result.RowsAffected() + if err != nil { + return false, fmt.Errorf("read data sync scheduler lease acquisition: %w", err) + } + return affected > 0, nil +} + +func (s *Store) ReleaseSchedulerLease(ctx context.Context, name, owner string) error { + if err := s.ensureOpen(); err != nil { + return err + } + name = strings.TrimSpace(name) + owner = strings.TrimSpace(owner) + if name == "" || owner == "" { + return errors.New("scheduler lease requires name and owner") + } + if _, err := s.db.ExecContext(ctx, `DELETE FROM data_sync_scheduler_leases WHERE name = ? AND owner_id = ?`, name, owner); err != nil { + return fmt.Errorf("release data sync scheduler lease: %w", err) + } + return nil +} + +func (s *Store) AppendRunEvent(ctx context.Context, event RunEvent) (RunEvent, error) { + if err := s.ensureOpen(); err != nil { + return RunEvent{}, err + } + event.RunID = strings.TrimSpace(event.RunID) + if event.RunID == "" || event.Type == "" { + return RunEvent{}, errors.New("data sync run event requires runId and type") + } + if !validJSONOrEmpty(event.Payload) { + return RunEvent{}, errors.New("data sync run event payload must be valid JSON") + } + if event.CreatedAt <= 0 { + event.CreatedAt = time.Now().UnixMilli() + } + const insert = `INSERT INTO data_sync_run_events( + run_id, sequence, job_id, event_type, status, current_item, total_items, table_name, stage, message, payload_json, created_at + ) SELECT run.id, + COALESCE((SELECT MAX(existing.sequence) + 1 FROM data_sync_run_events AS existing WHERE existing.run_id = run.id), 1), + run.job_id, ?, ?, ?, ?, ?, ?, ?, ?, ? + FROM data_sync_runs AS run WHERE run.id = ? + RETURNING sequence, job_id` + for attempt := 0; attempt < 8; attempt++ { + err := s.db.QueryRowContext(ctx, insert, event.Type, event.Status, event.Current, event.Total, + strings.TrimSpace(event.Table), strings.TrimSpace(event.Stage), event.Message, nullableBytes(event.Payload), + event.CreatedAt, event.RunID).Scan(&event.Sequence, &event.JobID) + switch { + case err == nil: + return event, nil + case errors.Is(err, sql.ErrNoRows): + return RunEvent{}, ErrNotFound + case strings.Contains(strings.ToLower(err.Error()), "unique constraint failed"): + continue + default: + return RunEvent{}, fmt.Errorf("append data sync run event: %w", err) + } + } + return RunEvent{}, errors.New("append data sync run event: sequence contention did not settle") +} + +func (s *Store) ListRunEvents(ctx context.Context, runID string, afterSequence int64, limit int) ([]RunEvent, error) { + if err := s.ensureOpen(); err != nil { + return nil, err + } + if afterSequence < 0 { + afterSequence = 0 + } + if limit < 1 { + limit = 200 + } + if limit > 2000 { + limit = 2000 + } + rows, err := s.db.QueryContext(ctx, `SELECT run_id, job_id, sequence, event_type, status, current_item, + total_items, table_name, stage, message, payload_json, created_at + FROM data_sync_run_events WHERE run_id = ? AND sequence > ? ORDER BY sequence LIMIT ?`, + strings.TrimSpace(runID), afterSequence, limit) + if err != nil { + return nil, fmt.Errorf("list data sync run events: %w", err) + } + defer rows.Close() + result := make([]RunEvent, 0) + for rows.Next() { + var event RunEvent + var payload []byte + if err := rows.Scan(&event.RunID, &event.JobID, &event.Sequence, &event.Type, &event.Status, + &event.Current, &event.Total, &event.Table, &event.Stage, &event.Message, &payload, &event.CreatedAt); err != nil { + return nil, fmt.Errorf("scan data sync run event: %w", err) + } + event.Payload = cloneRaw(payload) + result = append(result, event) + } + return result, rows.Err() +} + +const runColumns = `id, job_id, owner_token, job_revision, trigger_kind, status, started_at, finished_at, + parent_run_id, attempt, queued_at, heartbeat_at, current_item, total_items, table_name, stage, + rows_inserted, rows_updated, rows_deleted, rows_failed, message, resumable, definition_snapshot, + source_fingerprint, target_fingerprint, created_at, updated_at` + +const runSelect = `SELECT ` + runColumns + ` FROM data_sync_runs` + +const errorRowColumns = `id, run_id, job_id, source_table, target_table, operation, source_key_json, payload_json, + payload_policy, payload_hash, payload_size, error_text, error_code, error_class, attempts, status, + retry_owner, retry_lease_expires_at, created_at, updated_at` + +type rowScanner interface { + Scan(dest ...any) error +} + +func scanJob(scanner rowScanner) (JobDefinition, error) { + var payload []byte + var nextRunAt, lastScheduledAt, archivedAt int64 + if err := scanner.Scan(&payload, &nextRunAt, &lastScheduledAt, &archivedAt); errors.Is(err, sql.ErrNoRows) { + return JobDefinition{}, ErrNotFound + } else if err != nil { + return JobDefinition{}, fmt.Errorf("scan data sync job: %w", err) + } + var definition JobDefinition + if err := json.Unmarshal(payload, &definition); err != nil { + return JobDefinition{}, fmt.Errorf("decode data sync job: %w", err) + } + definition.NextRunAt = nextRunAt + definition.LastScheduledAt = lastScheduledAt + definition.ArchivedAt = archivedAt + return definition, nil +} + +func scanJobs(rows *sql.Rows) ([]JobDefinition, error) { + result := make([]JobDefinition, 0) + for rows.Next() { + definition, err := scanJob(rows) + if err != nil { + return nil, err + } + result = append(result, definition) + } + return result, rows.Err() +} + +func scanRun(scanner rowScanner) (RunRecord, error) { + var run RunRecord + var resumable int + var snapshot []byte + err := scanner.Scan(&run.ID, &run.JobID, &run.OwnerToken, &run.JobRevision, &run.Trigger, &run.Status, + &run.StartedAt, &run.FinishedAt, &run.ParentRunID, &run.Attempt, &run.QueuedAt, &run.HeartbeatAt, + &run.Current, &run.Total, &run.Table, &run.Stage, + &run.RowsInserted, &run.RowsUpdated, &run.RowsDeleted, &run.RowsFailed, &run.Message, + &resumable, &snapshot, &run.SourceFingerprint, &run.TargetFingerprint, &run.CreatedAt, &run.UpdatedAt) + if errors.Is(err, sql.ErrNoRows) { + return RunRecord{}, ErrNotFound + } + if err != nil { + return RunRecord{}, fmt.Errorf("scan data sync run: %w", err) + } + run.Resumable = resumable != 0 + run.DefinitionSnapshot = cloneRaw(snapshot) + return run, nil +} + +func scanErrorRow(scanner rowScanner) (ErrorRow, error) { + var row ErrorRow + var sourceKey, payload []byte + err := scanner.Scan(&row.ID, &row.RunID, &row.JobID, &row.SourceTable, &row.TargetTable, + &row.Operation, &sourceKey, &payload, &row.PayloadPolicy, &row.PayloadHash, &row.PayloadSize, + &row.Error, &row.ErrorCode, &row.ErrorClass, &row.Attempts, &row.Status, &row.RetryOwner, + &row.RetryLeaseExpiresAt, &row.CreatedAt, &row.UpdatedAt) + if errors.Is(err, sql.ErrNoRows) { + return ErrorRow{}, ErrNotFound + } + if err != nil { + return ErrorRow{}, fmt.Errorf("scan data sync error row: %w", err) + } + row.SourceKey = cloneRaw(sourceKey) + row.Payload = cloneRaw(payload) + return row, nil +} + +func (s *Store) ensureOpen() error { + if s == nil || s.db == nil { + return ErrClosed + } + return nil +} + +func requireAffected(result sql.Result) error { + affected, err := result.RowsAffected() + if err != nil { + return err + } + if affected == 0 { + return ErrNotFound + } + return nil +} + +func requireOwnedAffected(result sql.Result) error { + affected, err := result.RowsAffected() + if err != nil { + return err + } + if affected == 0 { + return ErrRunOwnershipLost + } + return nil +} + +func requireUnownedAffected(result sql.Result) error { + affected, err := result.RowsAffected() + if err != nil { + return err + } + if affected == 0 { + return ErrRunOwnershipLost + } + return nil +} + +func boolInt(value bool) int { + if value { + return 1 + } + return 0 +} + +func nullableBytes(raw json.RawMessage) any { + if len(raw) == 0 { + return nil + } + return []byte(raw) +} + +func cloneRaw(raw []byte) json.RawMessage { + if len(raw) == 0 { + return nil + } + return append(json.RawMessage(nil), raw...) +} + +func sqliteDSN(path string) string { + uriPath := filepath.ToSlash(path) + dsn := &url.URL{Scheme: "file", Path: uriPath} + if runtime.GOOS == "windows" { + if strings.HasPrefix(uriPath, "//") { + withoutPrefix := strings.TrimPrefix(uriPath, "//") + if separator := strings.IndexByte(withoutPrefix, '/'); separator >= 0 { + dsn.Host = withoutPrefix[:separator] + dsn.Path = withoutPrefix[separator:] + } + } else if !strings.HasPrefix(uriPath, "/") { + dsn.Path = "/" + uriPath + } + } + query := url.Values{} + for _, pragma := range []string{"busy_timeout(5000)", "foreign_keys(ON)", "synchronous(FULL)", "journal_mode(WAL)"} { + query.Add("_pragma", pragma) + } + query.Set("_txlock", "immediate") + dsn.RawQuery = query.Encode() + return dsn.String() +} diff --git a/internal/syncjob/store_test.go b/internal/syncjob/store_test.go new file mode 100644 index 00000000..9b0c885a --- /dev/null +++ b/internal/syncjob/store_test.go @@ -0,0 +1,296 @@ +package syncjob + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sync" + "testing" + "time" +) + +func TestStoreSchedulerLeaseAllowsOnlyTheOwnerUntilExpiry(t *testing.T) { + store := openTestStore(t) + now := time.Now().Truncate(time.Millisecond) + acquired, err := store.AcquireSchedulerLease(context.Background(), "scheduler", "owner-a", now, time.Second) + if err != nil || !acquired { + t.Fatalf("owner-a acquire = %v, %v", acquired, err) + } + acquired, err = store.AcquireSchedulerLease(context.Background(), "scheduler", "owner-b", now.Add(500*time.Millisecond), time.Second) + if err != nil { + t.Fatalf("owner-b acquire before expiry: %v", err) + } + if acquired { + t.Fatal("owner-b acquired a live lease") + } + acquired, err = store.AcquireSchedulerLease(context.Background(), "scheduler", "owner-b", now.Add(time.Second), time.Second) + if err != nil || !acquired { + t.Fatalf("owner-b takeover = %v, %v", acquired, err) + } + if err := store.ReleaseSchedulerLease(context.Background(), "scheduler", "owner-a"); err != nil { + t.Fatalf("old owner release: %v", err) + } + acquired, err = store.AcquireSchedulerLease(context.Background(), "scheduler", "owner-c", now.Add(1500*time.Millisecond), time.Second) + if err != nil { + t.Fatalf("owner-c acquire while owner-b live: %v", err) + } + if acquired { + t.Fatal("old owner release removed the replacement lease") + } +} + +func TestStorePersistsConcurrentRunEventsWithContiguousSequences(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "queue") + snapshot, err := json.Marshal(definition) + if err != nil { + t.Fatalf("marshal definition: %v", err) + } + run, err := store.CreateRun(context.Background(), RunRecord{ + JobID: definition.ID, + JobRevision: definition.Revision, + DefinitionSnapshot: snapshot, + }) + if err != nil { + t.Fatalf("create run: %v", err) + } + + const count = 32 + errorsSeen := make(chan error, count) + var wait sync.WaitGroup + for index := 0; index < count; index++ { + index := index + wait.Add(1) + go func() { + defer wait.Done() + _, appendErr := store.AppendRunEvent(context.Background(), RunEvent{ + RunID: run.ID, + Type: RunEventLog, + Message: fmt.Sprintf("event-%d", index), + }) + if appendErr != nil { + errorsSeen <- appendErr + } + }() + } + wait.Wait() + close(errorsSeen) + for appendErr := range errorsSeen { + t.Errorf("append event: %v", appendErr) + } + events, err := store.ListRunEvents(context.Background(), run.ID, 0, count) + if err != nil { + t.Fatalf("list events: %v", err) + } + if len(events) != count { + t.Fatalf("event count = %d, want %d", len(events), count) + } + for index, event := range events { + if event.Sequence != int64(index+1) { + t.Fatalf("event sequence at %d = %d, want %d", index, event.Sequence, index+1) + } + } +} + +func TestStorePersistsIncompleteDraftButManagerWillNotRunIt(t *testing.T) { + store := openTestStore(t) + draft, err := store.PutJob(context.Background(), JobDefinition{ + Name: "unfinished sync", + Lifecycle: JobLifecycleDraft, + }) + if err != nil { + t.Fatalf("persist draft: %v", err) + } + if draft.Enabled || draft.NextRunAt != 0 || draft.Lifecycle != JobLifecycleDraft { + t.Fatalf("normalized draft = %#v", draft) + } + manager := newTestManager(t, store, ExecutorFunc(func(context.Context, ExecutionRequest, RunReporter) (ExecutionOutcome, error) { + t.Fatal("draft executor must not run") + return ExecutionOutcome{}, nil + })) + if _, err := manager.StartRun(context.Background(), draft.ID); !errors.Is(err, ErrJobDisabled) { + t.Fatalf("start draft error = %v, want ErrJobDisabled", err) + } + if _, err := store.PutJob(context.Background(), JobDefinition{ + Name: "invalid ready job", + Lifecycle: JobLifecycleReady, + }); err == nil { + t.Fatal("ready job without endpoints and mappings was persisted") + } +} + +func TestStoreResetCheckpointRejectsActiveRun(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "forbid") + snapshot, err := json.Marshal(definition) + if err != nil { + t.Fatalf("marshal definition: %v", err) + } + run, err := store.CreateRun(context.Background(), RunRecord{ + JobID: definition.ID, + JobRevision: definition.Revision, + Status: RunStatusRunning, + DefinitionSnapshot: snapshot, + }) + if err != nil { + t.Fatalf("create running run: %v", err) + } + if _, err := store.PutCheckpoint(context.Background(), Checkpoint{ + Version: 1, + Kind: "watermark", + JobID: definition.ID, + RunID: run.ID, + DefinitionRevision: definition.Revision, + Table: "orders", + Phase: "batch_committed", + CursorType: "watermark_map", + Cursor: json.RawMessage(`{"orders":{"id":42}}`), + }); err != nil { + t.Fatalf("put checkpoint: %v", err) + } + if err := store.ResetCheckpoint(context.Background(), definition.ID); !errors.Is(err, ErrRunAlreadyActive) { + t.Fatalf("reset with active run error = %v, want ErrRunAlreadyActive", err) + } + if _, err := store.CompleteRun(context.Background(), run.ID, RunStatusFailed, ExecutionOutcome{Resumable: true}, "failed", time.Now().UnixMilli()); err != nil { + t.Fatalf("complete run: %v", err) + } + if err := store.ResetCheckpoint(context.Background(), definition.ID); err != nil { + t.Fatalf("reset checkpoint: %v", err) + } + if err := store.ResetCheckpoint(context.Background(), definition.ID); !errors.Is(err, ErrNotFound) { + t.Fatalf("second reset error = %v, want ErrNotFound", err) + } +} + +func TestManagerReadsAndDiscardsErrorRowWithOneWayCAS(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "queue") + snapshot, err := json.Marshal(definition) + if err != nil { + t.Fatalf("marshal definition: %v", err) + } + run, err := store.CreateRun(context.Background(), RunRecord{ + JobID: definition.ID, + JobRevision: definition.Revision, + Status: RunStatusFailed, + DefinitionSnapshot: snapshot, + }) + if err != nil { + t.Fatalf("create run: %v", err) + } + row, err := store.AppendErrorRow(context.Background(), ErrorRow{ + RunID: run.ID, + JobID: definition.ID, + Error: "duplicate key", + }) + if err != nil { + t.Fatalf("append error row: %v", err) + } + manager := newTestManager(t, store, ExecutorFunc(func(context.Context, ExecutionRequest, RunReporter) (ExecutionOutcome, error) { + return ExecutionOutcome{}, nil + })) + read, err := manager.GetErrorRow(context.Background(), row.ID) + if err != nil || read.ID != row.ID || read.Status != ErrorRowPending { + t.Fatalf("get error row = %#v, err=%v", read, err) + } + if err := manager.RecordErrorRowRetryFailure(context.Background(), row.ID); err != nil { + t.Fatalf("record retry failure: %v", err) + } + read, err = manager.GetErrorRow(context.Background(), row.ID) + if err != nil || read.Status != ErrorRowPending || read.Attempts != 1 { + t.Fatalf("pending retried error row = %#v, err=%v", read, err) + } + if err := manager.DiscardErrorRow(context.Background(), row.ID); err != nil { + t.Fatalf("discard error row: %v", err) + } + discarded, err := manager.GetErrorRow(context.Background(), row.ID) + if err != nil || discarded.Status != ErrorRowDiscarded { + t.Fatalf("discarded error row = %#v, err=%v", discarded, err) + } + if err := manager.DiscardErrorRow(context.Background(), row.ID); !errors.Is(err, ErrErrorRowStateConflict) { + t.Fatalf("repeat discard error = %v, want ErrErrorRowStateConflict", err) + } + if err := manager.RecordErrorRowRetryFailure(context.Background(), row.ID); !errors.Is(err, ErrErrorRowStateConflict) { + t.Fatalf("discarded retry failure error = %v, want ErrErrorRowStateConflict", err) + } + if err := store.UpdateErrorRowStatus(context.Background(), row.ID, ErrorRowResolved, true); !errors.Is(err, ErrErrorRowStateConflict) { + t.Fatalf("discarded to resolved error = %v, want ErrErrorRowStateConflict", err) + } + resolvedRow, err := store.AppendErrorRow(context.Background(), ErrorRow{ + RunID: run.ID, + JobID: definition.ID, + Error: "timeout", + }) + if err != nil { + t.Fatalf("append resolvable error row: %v", err) + } + if err := manager.ResolveErrorRow(context.Background(), resolvedRow.ID, true); err != nil { + t.Fatalf("resolve error row: %v", err) + } + resolved, err := manager.GetErrorRow(context.Background(), resolvedRow.ID) + if err != nil || resolved.Status != ErrorRowResolved || resolved.Attempts != 1 { + t.Fatalf("resolved error row = %#v, err=%v", resolved, err) + } + if err := manager.DiscardErrorRow(context.Background(), resolvedRow.ID); !errors.Is(err, ErrErrorRowStateConflict) { + t.Fatalf("resolved to discarded error = %v, want ErrErrorRowStateConflict", err) + } +} + +func TestStoreErrorRowRetryClaimFencesTransitionsAndRecoversExpiredLease(t *testing.T) { + store := openTestStore(t) + definition := putTestJob(t, store, "queue") + run := createStoredRun(t, store, definition, RunStatusFailed) + row, err := store.AppendErrorRow(context.Background(), ErrorRow{ + RunID: run.ID, + JobID: definition.ID, + Error: "duplicate key", + }) + if err != nil { + t.Fatalf("append error row: %v", err) + } + now := time.Now().UnixMilli() + claimed, err := store.ClaimErrorRowRetry(context.Background(), row.ID, now, time.Second) + if err != nil { + t.Fatalf("claim error row retry: %v", err) + } + if claimed.Status != ErrorRowRetrying || claimed.RetryOwner == "" || claimed.RetryLeaseExpiresAt != now+time.Second.Milliseconds() { + t.Fatalf("claimed error row = %#v", claimed) + } + if _, err := store.ClaimErrorRowRetry(context.Background(), row.ID, now+500, time.Second); !errors.Is(err, ErrErrorRowStateConflict) { + t.Fatalf("concurrent claim error = %v, want ErrErrorRowStateConflict", err) + } + if err := store.UpdateErrorRowStatus(context.Background(), row.ID, ErrorRowDiscarded, false); !errors.Is(err, ErrErrorRowStateConflict) { + t.Fatalf("discard retrying row error = %v, want ErrErrorRowStateConflict", err) + } + if err := store.ResolveErrorRowRetry(context.Background(), row.ID, "wrong-owner", now+600); !errors.Is(err, ErrErrorRowRetryOwnershipLost) { + t.Fatalf("wrong-owner resolution error = %v, want ErrErrorRowRetryOwnershipLost", err) + } + if err := store.RenewErrorRowRetry(context.Background(), row.ID, claimed.RetryOwner, now+500, time.Second); err != nil { + t.Fatalf("renew retry claim: %v", err) + } + if recovered, err := store.RecoverExpiredErrorRowRetries(context.Background(), now+time.Second.Milliseconds()+1); err != nil || recovered != 0 { + t.Fatalf("recover live renewed retry claims = %d, err=%v", recovered, err) + } + + recovered, err := store.RecoverExpiredErrorRowRetries(context.Background(), now+1501) + if err != nil || recovered != 1 { + t.Fatalf("recover expired retry claims = %d, err=%v", recovered, err) + } + pending, err := store.GetErrorRow(context.Background(), row.ID) + if err != nil || pending.Status != ErrorRowPending || pending.Attempts != 1 || pending.RetryOwner != "" || pending.RetryLeaseExpiresAt != 0 { + t.Fatalf("recovered retry row = %#v, err=%v", pending, err) + } + + reclaimed, err := store.ClaimErrorRowRetry(context.Background(), row.ID, now+2000, time.Second) + if err != nil { + t.Fatalf("reclaim recovered error row: %v", err) + } + if err := store.ResolveErrorRowRetry(context.Background(), row.ID, reclaimed.RetryOwner, now+2100); err != nil { + t.Fatalf("resolve reclaimed error row: %v", err) + } + resolved, err := store.GetErrorRow(context.Background(), row.ID) + if err != nil || resolved.Status != ErrorRowResolved || resolved.Attempts != 2 || resolved.RetryOwner != "" || resolved.RetryLeaseExpiresAt != 0 { + t.Fatalf("resolved retried row = %#v, err=%v", resolved, err) + } +} diff --git a/internal/syncjob/validation.go b/internal/syncjob/validation.go new file mode 100644 index 00000000..000516fd --- /dev/null +++ b/internal/syncjob/validation.go @@ -0,0 +1,593 @@ +package syncjob + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "time" +) + +const ( + defaultBatchSize = 1000 + maxBatchSize = 10000 + maxRetries = 10 + minScheduleInterval = 10 * time.Second + continuousRunPoll = 5 * time.Second +) + +var supportedTransforms = map[string]struct{}{ + "": {}, + "identity": {}, + "string": {}, + "int64": {}, + "float64": {}, + "bool": {}, + "timestamp": {}, + "date": {}, + "json": {}, + "lower": {}, + "upper": {}, + "trim": {}, + "constant": {}, + "coalesce": {}, +} + +func NormalizeDefinition(input JobDefinition) JobDefinition { + definition := input + if definition.Version == 0 { + definition.Version = CurrentDefinitionVersion + } + definition.ID = strings.TrimSpace(definition.ID) + definition.Name = strings.TrimSpace(definition.Name) + definition.Description = strings.TrimSpace(definition.Description) + if definition.Lifecycle == "" { + switch { + case definition.ArchivedAt > 0: + definition.Lifecycle = JobLifecycleArchived + case definition.Enabled: + definition.Lifecycle = JobLifecycleEnabled + default: + definition.Lifecycle = JobLifecycleReady + } + } + definition.Enabled = definition.Lifecycle == JobLifecycleEnabled + definition.Source = normalizeEndpoint(definition.Source) + definition.Target = normalizeEndpoint(definition.Target) + if definition.Kind == "" { + definition.Kind = JobKindReconcile + } + if definition.IncrementalMode == "" { + definition.IncrementalMode = IncrementalSnapshot + } + definition.SourceQuery = strings.TrimSpace(definition.SourceQuery) + if definition.CDC != nil { + definition.CDC.Adapter = strings.ToLower(strings.TrimSpace(definition.CDC.Adapter)) + definition.CDC.StartPosition = strings.ToLower(strings.TrimSpace(definition.CDC.StartPosition)) + definition.CDC.SlotName = strings.TrimSpace(definition.CDC.SlotName) + definition.CDC.PublicationName = strings.TrimSpace(definition.CDC.PublicationName) + } + if definition.Approval != nil { + definition.Approval.DefinitionHash = strings.TrimSpace(definition.Approval.DefinitionHash) + definition.Approval.TargetFingerprint = strings.TrimSpace(definition.Approval.TargetFingerprint) + definition.Approval.ApprovedByRuntime = strings.TrimSpace(definition.Approval.ApprovedByRuntime) + } + if definition.Schedule.Kind == "" { + definition.Schedule.Kind = ScheduleManual + } + if definition.Schedule.MisfirePolicy == "" { + definition.Schedule.MisfirePolicy = "skip" + } + definition.Schedule.CronExpression = strings.TrimSpace(definition.Schedule.CronExpression) + definition.Schedule.Timezone = strings.TrimSpace(definition.Schedule.Timezone) + if definition.Schedule.Timezone == "" { + definition.Schedule.Timezone = "Local" + } + if definition.ConcurrencyPolicy == "" { + definition.ConcurrencyPolicy = "forbid" + } + if definition.ResumePolicy == "" { + definition.ResumePolicy = "manual" + } + if definition.Options.Content == "" { + definition.Options.Content = "data" + } + if definition.Options.SyncMode == "" { + definition.Options.SyncMode = "insert_update" + } + if definition.Options.TargetTableStrategy == "" { + definition.Options.TargetTableStrategy = "existing_only" + } + if definition.Options.BatchSize == 0 { + definition.Options.BatchSize = defaultBatchSize + } + if definition.Options.ErrorPolicy == "" { + definition.Options.ErrorPolicy = ErrorPolicyStop + } + if definition.Options.RetryBackoffMillis == 0 { + definition.Options.RetryBackoffMillis = 500 + } + for index := range definition.Mappings { + mapping := &definition.Mappings[index] + mapping.SourceSchema = strings.TrimSpace(mapping.SourceSchema) + mapping.SourceTable = strings.TrimSpace(mapping.SourceTable) + mapping.TargetSchema = strings.TrimSpace(mapping.TargetSchema) + mapping.TargetTable = strings.TrimSpace(mapping.TargetTable) + mapping.TargetTableStrategy = strings.ToLower(strings.TrimSpace(mapping.TargetTableStrategy)) + mapping.Filter = strings.TrimSpace(mapping.Filter) + mapping.KeyColumns = normalizeUniqueStrings(mapping.KeyColumns) + for columnIndex := range mapping.Columns { + column := &mapping.Columns[columnIndex] + column.Source = strings.TrimSpace(column.Source) + column.Target = strings.TrimSpace(column.Target) + column.Transform.Kind = strings.ToLower(strings.TrimSpace(column.Transform.Kind)) + } + if mapping.Watermark != nil { + mapping.Watermark.Column = strings.TrimSpace(mapping.Watermark.Column) + mapping.Watermark.TieBreakerColumns = normalizeUniqueStrings(mapping.Watermark.TieBreakerColumns) + } + } + return definition +} + +func ValidateDefinition(input JobDefinition) error { + definition := NormalizeDefinition(input) + if err := validatePersistableDefinitionEnums(definition); err != nil { + return err + } + if definition.Source.ConnectionID == "" { + return errors.New("source saved connection is required") + } + if definition.Target.ConnectionID == "" { + return errors.New("target saved connection is required") + } + if definition.Approval != nil { + if definition.Approval.DefinitionHash == "" || definition.Approval.TargetFingerprint == "" || definition.Approval.ApprovedAt <= 0 || definition.Approval.ApprovedByRuntime == "" { + return errors.New("execution approval requires definitionHash, targetFingerprint, approvedAt, and approvedByRuntime") + } + } + switch definition.Kind { + case JobKindMigration, JobKindReconcile, JobKindQuerySink, JobKindCompare: + default: + return fmt.Errorf("unsupported data sync job kind %q", definition.Kind) + } + switch definition.IncrementalMode { + case IncrementalSnapshot, IncrementalWatermark, IncrementalCDC: + default: + return fmt.Errorf("unsupported incremental mode %q", definition.IncrementalMode) + } + if definition.Kind == JobKindQuerySink && definition.SourceQuery == "" { + return errors.New("query sink jobs require sourceQuery") + } + if definition.Kind != JobKindQuerySink && definition.SourceQuery != "" { + return errors.New("sourceQuery is only supported by query sink jobs") + } + if len(definition.Mappings) == 0 { + return errors.New("at least one table mapping is required") + } + if definition.Kind == JobKindQuerySink && len(definition.Mappings) != 1 { + return errors.New("query sink jobs require exactly one target mapping") + } + if (definition.Kind == JobKindQuerySink || definition.Kind == JobKindCompare) && definition.IncrementalMode != IncrementalSnapshot { + return fmt.Errorf("%s jobs only support snapshot execution", definition.Kind) + } + seenTargets := make(map[string]struct{}, len(definition.Mappings)) + enabledMappings := 0 + for index, mapping := range definition.Mappings { + if !mapping.Enabled { + continue + } + enabledMappings++ + if mapping.TargetTable == "" || (definition.Kind != JobKindQuerySink && mapping.SourceTable == "") { + return fmt.Errorf("table mapping %d requires a targetTable and a sourceTable unless this is a query sink", index+1) + } + switch mapping.TargetTableStrategy { + case "", "existing_only", "auto_create_if_missing", "smart": + default: + return fmt.Errorf("table mapping %s has unsupported targetTableStrategy %q", mapping.SourceTable, mapping.TargetTableStrategy) + } + targetKey := strings.ToLower(mapping.TargetSchema + "\x00" + mapping.TargetTable) + if _, exists := seenTargets[targetKey]; exists { + return fmt.Errorf("duplicate target table mapping %s", mapping.TargetTable) + } + seenTargets[targetKey] = struct{}{} + if err := validateColumnMappings(mapping); err != nil { + return fmt.Errorf("table mapping %s: %w", mapping.SourceTable, err) + } + if definition.IncrementalMode == IncrementalWatermark { + if mapping.Watermark == nil || strings.TrimSpace(mapping.Watermark.Column) == "" { + return fmt.Errorf("table mapping %s requires a watermark column", mapping.SourceTable) + } + } + if definition.IncrementalMode == IncrementalCDC && len(mapping.KeyColumns) == 0 { + return fmt.Errorf("table mapping %s requires stable keyColumns for CDC", mapping.SourceTable) + } + } + if enabledMappings == 0 { + return errors.New("at least one table mapping must be enabled") + } + if definition.IncrementalMode == IncrementalCDC { + if definition.CDC == nil || strings.TrimSpace(definition.CDC.Adapter) == "" { + return errors.New("CDC jobs require an explicit adapter") + } + switch definition.CDC.StartPosition { + case "", "checkpoint", "latest", "earliest": + default: + return fmt.Errorf("unsupported CDC start position %q", definition.CDC.StartPosition) + } + if definition.Options.TargetTableStrategy != "existing_only" { + return errors.New("CDC jobs require existing target tables") + } + } + if definition.Options.BatchSize < 1 || definition.Options.BatchSize > maxBatchSize { + return fmt.Errorf("batchSize must be between 1 and %d", maxBatchSize) + } + if definition.Options.MaxRetries < 0 || definition.Options.MaxRetries > maxRetries { + return fmt.Errorf("maxRetries must be between 0 and %d", maxRetries) + } + if definition.Options.RetryBackoffMillis < 0 || definition.Options.RetryBackoffMillis > int((5*time.Minute)/time.Millisecond) { + return errors.New("retryBackoffMillis must be between 0 and 300000") + } + switch definition.Options.ErrorPolicy { + case ErrorPolicyStop, ErrorPolicySkipRow: + default: + return fmt.Errorf("unsupported error policy %q", definition.Options.ErrorPolicy) + } + switch definition.Schedule.Kind { + case ScheduleManual: + case ScheduleOnce: + if definition.Schedule.RunAt <= 0 { + return errors.New("one-time schedules require runAt") + } + case ScheduleInterval: + if time.Duration(definition.Schedule.IntervalSeconds)*time.Second < minScheduleInterval { + return fmt.Errorf("scheduled interval must be at least %s", minScheduleInterval) + } + case ScheduleCron: + if _, err := parseCronSchedule(definition.Schedule.CronExpression, definition.Schedule.Timezone); err != nil { + return err + } + case ScheduleContinuous: + if definition.IncrementalMode != IncrementalCDC { + return errors.New("continuous trigger requires CDC incremental mode") + } + if definition.ConcurrencyPolicy != "forbid" { + return errors.New("continuous trigger requires forbid concurrency policy") + } + default: + return fmt.Errorf("unsupported schedule kind %q", definition.Schedule.Kind) + } + switch definition.Schedule.MisfirePolicy { + case "skip", "run_once", "catch_up": + default: + return fmt.Errorf("unsupported misfire policy %q", definition.Schedule.MisfirePolicy) + } + switch definition.ConcurrencyPolicy { + case "forbid", "queue": + default: + return fmt.Errorf("unsupported concurrency policy %q", definition.ConcurrencyPolicy) + } + switch definition.ResumePolicy { + case "never", "manual", "auto": + default: + return fmt.Errorf("unsupported resume policy %q", definition.ResumePolicy) + } + return nil +} + +func ValidatePersistableDefinition(input JobDefinition) error { + definition := NormalizeDefinition(input) + if err := validatePersistableDefinitionEnums(definition); err != nil { + return err + } + if definition.Lifecycle == JobLifecycleDraft || definition.Lifecycle == JobLifecycleArchived { + return nil + } + return ValidateDefinition(definition) +} + +func validatePersistableDefinitionEnums(definition JobDefinition) error { + if definition.Version != CurrentDefinitionVersion { + return fmt.Errorf("unsupported data sync job definition version %d", definition.Version) + } + if definition.Name == "" { + return errors.New("data sync job name is required") + } + switch definition.Lifecycle { + case JobLifecycleDraft, JobLifecycleReady, JobLifecycleEnabled, JobLifecyclePaused, JobLifecycleArchived: + default: + return fmt.Errorf("unsupported data sync job lifecycle %q", definition.Lifecycle) + } + switch definition.Kind { + case JobKindMigration, JobKindReconcile, JobKindQuerySink, JobKindCompare: + default: + return fmt.Errorf("unsupported data sync job kind %q", definition.Kind) + } + switch definition.IncrementalMode { + case IncrementalSnapshot, IncrementalWatermark, IncrementalCDC: + default: + return fmt.Errorf("unsupported incremental mode %q", definition.IncrementalMode) + } + switch definition.Schedule.Kind { + case ScheduleManual, ScheduleOnce, ScheduleInterval, ScheduleCron, ScheduleContinuous: + default: + return fmt.Errorf("unsupported schedule kind %q", definition.Schedule.Kind) + } + switch definition.Schedule.MisfirePolicy { + case "skip", "run_once", "catch_up": + default: + return fmt.Errorf("unsupported misfire policy %q", definition.Schedule.MisfirePolicy) + } + switch definition.ConcurrencyPolicy { + case "forbid", "queue": + default: + return fmt.Errorf("unsupported concurrency policy %q", definition.ConcurrencyPolicy) + } + switch definition.ResumePolicy { + case "never", "manual", "auto": + default: + return fmt.Errorf("unsupported resume policy %q", definition.ResumePolicy) + } + switch definition.Options.ErrorPolicy { + case ErrorPolicyStop, ErrorPolicySkipRow: + default: + return fmt.Errorf("unsupported error policy %q", definition.Options.ErrorPolicy) + } + return nil +} + +func validateColumnMappings(mapping TableMapping) error { + seenTargets := make(map[string]struct{}, len(mapping.Columns)) + for index, column := range mapping.Columns { + if column.Target == "" { + return fmt.Errorf("column mapping %d requires target", index+1) + } + if column.Source == "" && column.Transform.Kind != "constant" && len(bytes.TrimSpace(column.DefaultValue)) == 0 { + return fmt.Errorf("column mapping %s requires source, constant transform, or defaultValue", column.Target) + } + targetKey := strings.ToLower(column.Target) + if _, exists := seenTargets[targetKey]; exists { + return fmt.Errorf("duplicate target column %s", column.Target) + } + seenTargets[targetKey] = struct{}{} + if _, ok := supportedTransforms[column.Transform.Kind]; !ok { + return fmt.Errorf("unsupported transform %q", column.Transform.Kind) + } + if !validJSONOrEmpty(column.Transform.Argument) { + return fmt.Errorf("transform argument for %s is not valid JSON", column.Target) + } + if !validJSONOrEmpty(column.DefaultValue) { + return fmt.Errorf("default value for %s is not valid JSON", column.Target) + } + } + return nil +} + +func normalizeEndpoint(endpoint EndpointRef) EndpointRef { + endpoint.ConnectionID = strings.TrimSpace(endpoint.ConnectionID) + endpoint.ConnectionType = strings.ToLower(strings.TrimSpace(endpoint.ConnectionType)) + endpoint.ConnectionName = strings.TrimSpace(endpoint.ConnectionName) + endpoint.Database = strings.TrimSpace(endpoint.Database) + endpoint.Schema = strings.TrimSpace(endpoint.Schema) + endpoint.Fingerprint = strings.TrimSpace(endpoint.Fingerprint) + return endpoint +} + +func normalizeUniqueStrings(values []string) []string { + result := make([]string, 0, len(values)) + seen := make(map[string]struct{}, len(values)) + for _, value := range values { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + continue + } + key := strings.ToLower(trimmed) + if _, exists := seen[key]; exists { + continue + } + seen[key] = struct{}{} + result = append(result, trimmed) + } + return result +} + +func validJSONOrEmpty(raw json.RawMessage) bool { + trimmed := bytes.TrimSpace(raw) + return len(trimmed) == 0 || json.Valid(trimmed) +} + +func NextRunAt(definition JobDefinition, after time.Time) int64 { + definition = NormalizeDefinition(definition) + if !definition.Enabled { + return 0 + } + switch definition.Schedule.Kind { + case ScheduleOnce: + if definition.Schedule.RunAt > after.UnixMilli() { + return definition.Schedule.RunAt + } + return 0 + case ScheduleCron: + next, err := nextCronTime(definition.Schedule.CronExpression, definition.Schedule.Timezone, after) + if err != nil { + return 0 + } + return next.UnixMilli() + case ScheduleInterval: + if definition.Schedule.IntervalSeconds <= 0 { + return 0 + } + case ScheduleContinuous: + // Continuous jobs are reconciled by the leased scheduler. While a stream + // is active the forbid policy suppresses duplicates; after EOF/failure the + // next poll starts a fresh run from the durable CDC checkpoint. + return after.Add(continuousRunPoll).UnixMilli() + default: + return 0 + } + interval := time.Duration(definition.Schedule.IntervalSeconds) * time.Second + anchorMillis := definition.Schedule.AnchorAt + if anchorMillis <= 0 { + return after.Add(interval).UnixMilli() + } + anchor := time.UnixMilli(anchorMillis) + if after.Before(anchor) { + return anchor.UnixMilli() + } + steps := after.Sub(anchor)/interval + 1 + return anchor.Add(steps * interval).UnixMilli() +} + +type cronSchedule struct { + minutes map[int]struct{} + hours map[int]struct{} + daysOfMonth map[int]struct{} + months map[int]struct{} + daysOfWeek map[int]struct{} + anyDayOfMonth bool + anyDayOfWeek bool + location *time.Location +} + +func parseCronSchedule(expression, timezone string) (cronSchedule, error) { + parts := strings.Fields(strings.TrimSpace(expression)) + if len(parts) != 5 { + return cronSchedule{}, errors.New("cronExpression must contain five fields: minute hour day month weekday") + } + location, err := time.LoadLocation(strings.TrimSpace(timezone)) + if err != nil { + return cronSchedule{}, fmt.Errorf("invalid schedule timezone %q: %w", timezone, err) + } + minutes, _, err := parseCronField(parts[0], 0, 59, false) + if err != nil { + return cronSchedule{}, fmt.Errorf("invalid cron minute: %w", err) + } + hours, _, err := parseCronField(parts[1], 0, 23, false) + if err != nil { + return cronSchedule{}, fmt.Errorf("invalid cron hour: %w", err) + } + daysOfMonth, anyDayOfMonth, err := parseCronField(parts[2], 1, 31, false) + if err != nil { + return cronSchedule{}, fmt.Errorf("invalid cron day: %w", err) + } + months, _, err := parseCronField(parts[3], 1, 12, false) + if err != nil { + return cronSchedule{}, fmt.Errorf("invalid cron month: %w", err) + } + daysOfWeek, anyDayOfWeek, err := parseCronField(parts[4], 0, 7, true) + if err != nil { + return cronSchedule{}, fmt.Errorf("invalid cron weekday: %w", err) + } + return cronSchedule{ + minutes: minutes, + hours: hours, + daysOfMonth: daysOfMonth, + months: months, + daysOfWeek: daysOfWeek, + anyDayOfMonth: anyDayOfMonth, + anyDayOfWeek: anyDayOfWeek, + location: location, + }, nil +} + +func nextCronTime(expression, timezone string, after time.Time) (time.Time, error) { + schedule, err := parseCronSchedule(expression, timezone) + if err != nil { + return time.Time{}, err + } + candidate := after.In(schedule.location).Truncate(time.Minute).Add(time.Minute) + const maxMinutes = 366 * 24 * 60 * 5 + for checked := 0; checked < maxMinutes; checked++ { + if schedule.matches(candidate) { + return candidate, nil + } + candidate = candidate.Add(time.Minute) + } + return time.Time{}, errors.New("cronExpression has no execution time within five years") +} + +func (schedule cronSchedule) matches(candidate time.Time) bool { + if _, ok := schedule.minutes[candidate.Minute()]; !ok { + return false + } + if _, ok := schedule.hours[candidate.Hour()]; !ok { + return false + } + if _, ok := schedule.months[int(candidate.Month())]; !ok { + return false + } + _, dayMatches := schedule.daysOfMonth[candidate.Day()] + _, weekdayMatches := schedule.daysOfWeek[int(candidate.Weekday())] + switch { + case schedule.anyDayOfMonth && schedule.anyDayOfWeek: + return true + case schedule.anyDayOfMonth: + return weekdayMatches + case schedule.anyDayOfWeek: + return dayMatches + default: + return dayMatches || weekdayMatches + } +} + +func parseCronField(spec string, minValue, maxValue int, normalizeSunday bool) (map[int]struct{}, bool, error) { + spec = strings.TrimSpace(spec) + if spec == "" { + return nil, false, errors.New("field is empty") + } + values := make(map[int]struct{}) + any := spec == "*" + for _, item := range strings.Split(spec, ",") { + item = strings.TrimSpace(item) + if item == "" { + return nil, false, errors.New("field contains an empty list item") + } + step := 1 + base := item + if slash := strings.IndexByte(item, '/'); slash >= 0 { + base = item[:slash] + parsedStep, err := strconv.Atoi(item[slash+1:]) + if err != nil || parsedStep <= 0 { + return nil, false, fmt.Errorf("invalid step %q", item[slash+1:]) + } + step = parsedStep + } + start, end := minValue, maxValue + switch { + case base == "*": + case strings.Contains(base, "-"): + bounds := strings.Split(base, "-") + if len(bounds) != 2 { + return nil, false, fmt.Errorf("invalid range %q", base) + } + var err error + start, err = strconv.Atoi(bounds[0]) + if err != nil { + return nil, false, fmt.Errorf("invalid range start %q", bounds[0]) + } + end, err = strconv.Atoi(bounds[1]) + if err != nil { + return nil, false, fmt.Errorf("invalid range end %q", bounds[1]) + } + default: + value, err := strconv.Atoi(base) + if err != nil { + return nil, false, fmt.Errorf("invalid value %q", base) + } + start, end = value, value + } + if start < minValue || end > maxValue || start > end { + return nil, false, fmt.Errorf("value %d-%d is outside %d-%d", start, end, minValue, maxValue) + } + for value := start; value <= end; value += step { + if normalizeSunday && value == 7 { + values[0] = struct{}{} + } else { + values[value] = struct{}{} + } + } + } + return values, any, nil +} diff --git a/internal/syncjob/validation_test.go b/internal/syncjob/validation_test.go new file mode 100644 index 00000000..254062e4 --- /dev/null +++ b/internal/syncjob/validation_test.go @@ -0,0 +1,105 @@ +package syncjob + +import ( + "strings" + "testing" + "time" +) + +func validValidationTestDefinition() JobDefinition { + return JobDefinition{ + Name: "orders sync", + Lifecycle: JobLifecycleReady, + Kind: JobKindReconcile, + IncrementalMode: IncrementalSnapshot, + Source: EndpointRef{ConnectionID: "source"}, + Target: EndpointRef{ConnectionID: "target"}, + Mappings: []TableMapping{{ + SourceTable: "orders", + TargetTable: "orders_archive", + Enabled: true, + }}, + } +} + +func TestValidateDefinitionRejectsUnsupportedPerMappingTargetStrategy(t *testing.T) { + definition := validValidationTestDefinition() + definition.Mappings[0].TargetTableStrategy = "drop_and_replace" + err := ValidateDefinition(definition) + if err == nil || !strings.Contains(err.Error(), "targetTableStrategy") { + t.Fatalf("ValidateDefinition error = %v, want targetTableStrategy error", err) + } +} + +func TestNormalizeDefinitionPreservesDisabledMapping(t *testing.T) { + definition := validValidationTestDefinition() + definition.Mappings[0].Enabled = false + normalized := NormalizeDefinition(definition) + if normalized.Mappings[0].Enabled { + t.Fatal("normalization must not re-enable an explicitly disabled mapping") + } +} + +func TestValidateDefinitionIgnoresIncompleteDisabledMapping(t *testing.T) { + definition := validValidationTestDefinition() + definition.Mappings = append(definition.Mappings, TableMapping{Enabled: false}) + + if err := ValidateDefinition(definition); err != nil { + t.Fatalf("disabled draft mapping was rejected: %v", err) + } +} + +func TestValidateDefinitionAllowsQuerySinkWithoutSyntheticSourceTable(t *testing.T) { + definition := validValidationTestDefinition() + definition.Kind = JobKindQuerySink + definition.SourceQuery = "SELECT id, total FROM orders WHERE exported = false" + definition.Mappings[0].SourceTable = "" + if err := ValidateDefinition(definition); err != nil { + t.Fatalf("query sink definition was rejected: %v", err) + } +} + +func TestValidateDefinitionRejectsIncrementalCompare(t *testing.T) { + definition := validValidationTestDefinition() + definition.Kind = JobKindCompare + definition.IncrementalMode = IncrementalWatermark + definition.Mappings[0].Watermark = &WatermarkSpec{Column: "updated_at", TieBreakerColumns: []string{"id"}} + err := ValidateDefinition(definition) + if err == nil || !strings.Contains(err.Error(), "only support snapshot") { + t.Fatalf("ValidateDefinition error = %v, want compare snapshot-only error", err) + } +} + +func TestContinuousCDCJobsAreScheduledAndForbidOverlap(t *testing.T) { + definition := validValidationTestDefinition() + definition.Lifecycle = JobLifecycleEnabled + definition.IncrementalMode = IncrementalCDC + definition.CDC = &CDCSpec{Adapter: "mongodb-change-stream", StartPosition: "checkpoint"} + definition.Mappings[0].KeyColumns = []string{"id"} + definition.Schedule = ScheduleSpec{Kind: ScheduleContinuous} + definition.ConcurrencyPolicy = "forbid" + + if err := ValidateDefinition(definition); err != nil { + t.Fatalf("continuous CDC definition was rejected: %v", err) + } + after := time.UnixMilli(1_700_000_000_000) + if got, want := NextRunAt(definition, after), after.Add(continuousRunPoll).UnixMilli(); got != want { + t.Fatalf("NextRunAt() = %d, want %d", got, want) + } + + definition.ConcurrencyPolicy = "queue" + err := ValidateDefinition(definition) + if err == nil || !strings.Contains(err.Error(), "forbid concurrency") { + t.Fatalf("ValidateDefinition error = %v, want continuous overlap error", err) + } +} + +func TestValidateDefinitionRejectsCDCWithoutStableKeys(t *testing.T) { + definition := validValidationTestDefinition() + definition.IncrementalMode = IncrementalCDC + definition.CDC = &CDCSpec{Adapter: "mongodb-change-stream", StartPosition: "latest"} + err := ValidateDefinition(definition) + if err == nil || !strings.Contains(err.Error(), "stable keyColumns") { + t.Fatalf("ValidateDefinition error = %v, want CDC key error", err) + } +}