feat(data-import): 重构大文件导入与安全任务运行时

- SQL、CSV、JSON 与 XLSX 改为资源受限的流式解析,并支持压缩、编码和方言预检

- 增加遇错停止/继续策略、持久任务、错误行导出、源文件身份与安全取消

- 为数据库驱动补齐上下文事务和写入结果未知语义,避免失败重放与脏会话复用
This commit is contained in:
Syngnat
2026-08-08 20:51:23 +08:00
parent 4df879ccde
commit 390a249105
81 changed files with 15267 additions and 625 deletions

View File

@@ -12,7 +12,11 @@ import (
"strings"
"sync"
"testing"
"time"
"GoNavi-Wails/internal/connection"
"GoNavi-Wails/shared/i18n"
"github.com/golang-sql/sqlexp"
_ "modernc.org/sqlite"
)
@@ -115,6 +119,103 @@ type fakeSQLServerExecResult struct {
rowErr error
}
type sqlServerApplyChangesContextState struct {
execStarted chan struct{}
execRelease chan struct{}
}
type sqlServerApplyChangesContextConnector struct {
state *sqlServerApplyChangesContextState
}
type sqlServerApplyChangesContextDriver struct{}
type sqlServerApplyChangesContextConn struct {
state *sqlServerApplyChangesContextState
}
type sqlServerApplyChangesContextTx struct{}
func (c sqlServerApplyChangesContextConnector) Connect(context.Context) (driver.Conn, error) {
return &sqlServerApplyChangesContextConn{state: c.state}, nil
}
func (sqlServerApplyChangesContextConnector) Driver() driver.Driver {
return sqlServerApplyChangesContextDriver{}
}
func (sqlServerApplyChangesContextDriver) Open(string) (driver.Conn, error) {
return nil, errors.New("use connector")
}
func (*sqlServerApplyChangesContextConn) Prepare(string) (driver.Stmt, error) {
return nil, driver.ErrSkip
}
func (*sqlServerApplyChangesContextConn) Close() error { return nil }
func (*sqlServerApplyChangesContextConn) Begin() (driver.Tx, error) {
return nil, errors.New("legacy Begin must not be used")
}
func (*sqlServerApplyChangesContextConn) BeginTx(context.Context, driver.TxOptions) (driver.Tx, error) {
return sqlServerApplyChangesContextTx{}, nil
}
func (c *sqlServerApplyChangesContextConn) ExecContext(ctx context.Context, _ string, _ []driver.NamedValue) (driver.Result, error) {
select {
case c.state.execStarted <- struct{}{}:
default:
}
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-c.state.execRelease:
return nil, errors.New("execution released without cancellation")
}
}
func (sqlServerApplyChangesContextTx) Commit() error { return nil }
func (sqlServerApplyChangesContextTx) Rollback() error { return nil }
func TestSQLServerApplyChangesContextCancelsInFlightBatchInsert(t *testing.T) {
state := &sqlServerApplyChangesContextState{
execStarted: make(chan struct{}, 1),
execRelease: make(chan struct{}),
}
dbConn := sql.OpenDB(sqlServerApplyChangesContextConnector{state: state})
t.Cleanup(func() { _ = dbConn.Close() })
ctx, cancel := context.WithCancel(context.Background())
errCh := make(chan error, 1)
go func() {
errCh <- (&SqlServerDB{conn: dbConn}).ApplyChangesContext(ctx, "dbo.orders", connection.ChangeSet{
Inserts: []map[string]interface{}{{"id": 42, "status": "pending"}},
})
}()
select {
case <-state.execStarted:
cancel()
case <-time.After(time.Second):
cancel()
close(state.execRelease)
t.Fatal("ApplyChangesContext did not reach the context-aware batch insert path")
}
select {
case err := <-errCh:
if err == nil || !strings.Contains(err.Error(), context.Canceled.Error()) {
t.Fatalf("ApplyChangesContext error = %v, want context cancellation", err)
}
case <-time.After(time.Second):
close(state.execRelease)
t.Fatal("ApplyChangesContext did not return after cancellation")
}
}
var _ BatchApplierContext = (*SqlServerDB)(nil)
func (r fakeSQLServerExecResult) LastInsertId() (int64, error) {
return 0, errors.New("not implemented")
}
@@ -341,4 +442,3 @@ func TestSQLServerMetadataErrorsUseCurrentLanguage(t *testing.T) {
})
}
}