feat(data-import): 重构大文件导入与安全任务运行时

- SQL、CSV、JSON 与 XLSX 改为资源受限的流式解析,并支持压缩、编码和方言预检

- 增加遇错停止/继续策略、持久任务、错误行导出、源文件身份与安全取消

- 为数据库驱动补齐上下文事务和写入结果未知语义,避免失败重放与脏会话复用
This commit is contained in:
Syngnat
2026-08-08 20:51:23 +08:00
parent 4df879ccde
commit 390a249105
81 changed files with 15267 additions and 625 deletions

View File

@@ -840,15 +840,20 @@ ORDER BY tr.name`,
}
func (s *SqlServerDB) ApplyChanges(tableName string, changes connection.ChangeSet) error {
return s.ApplyChangesContext(context.Background(), tableName, changes)
}
func (s *SqlServerDB) ApplyChangesContext(ctx context.Context, tableName string, changes connection.ChangeSet) (err error) {
if s.conn == nil {
return fmt.Errorf("连接未打开")
}
tx, err := s.conn.Begin()
tx, err := s.conn.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
transactionCommitted := false
defer func() { rollbackUnfinishedWriteTransaction(tx, transactionCommitted, &err) }()
quoteIdent := func(name string) string {
n := strings.TrimSpace(name)
@@ -883,7 +888,7 @@ func (s *SqlServerDB) ApplyChanges(tableName string, changes connection.ChangeSe
continue
}
query := fmt.Sprintf("DELETE FROM %s WHERE %s", qualifiedTable, strings.Join(wheres, " AND "))
if _, err := tx.Exec(query, args...); err != nil {
if _, err := tx.ExecContext(ctx, query, args...); err != nil {
return fmt.Errorf("删除失败:%v", err)
}
}
@@ -916,7 +921,7 @@ func (s *SqlServerDB) ApplyChanges(tableName string, changes connection.ChangeSe
}
query := fmt.Sprintf("UPDATE %s SET %s WHERE %s", qualifiedTable, strings.Join(sets, ", "), strings.Join(wheres, " AND "))
if _, err := tx.Exec(query, args...); err != nil {
if _, err := tx.ExecContext(ctx, query, args...); err != nil {
return fmt.Errorf("更新失败:%v", err)
}
}
@@ -932,12 +937,16 @@ func (s *SqlServerDB) ApplyChanges(tableName string, changes connection.ChangeSe
return sql.Named(fmt.Sprintf("p%d", idx), value)
},
Exec: func(query string, args ...interface{}) (sql.Result, error) {
return tx.Exec(query, args...)
return tx.ExecContext(ctx, query, args...)
},
MaxArgs: sqlServerBatchInsertArgs,
}); err != nil {
return err
}
return tx.Commit()
if err := commitWriteTransaction(tx); err != nil {
return err
}
transactionCommitted = true
return nil
}