feat(data-import): 重构大文件导入与安全任务运行时

- SQL、CSV、JSON 与 XLSX 改为资源受限的流式解析,并支持压缩、编码和方言预检

- 增加遇错停止/继续策略、持久任务、错误行导出、源文件身份与安全取消

- 为数据库驱动补齐上下文事务和写入结果未知语义,避免失败重放与脏会话复用
This commit is contained in:
Syngnat
2026-08-08 20:51:23 +08:00
parent 4df879ccde
commit 390a249105
81 changed files with 15267 additions and 625 deletions

View File

@@ -39,3 +39,68 @@ func TestSQLStreamSplitterPreservesExecutableMySQLComment(t *testing.T) {
t.Fatalf("expected statements %#v, got %#v", want, got)
}
}
func TestSQLStreamSplitterTransactionBeginFormsAcrossChunkBoundaries(t *testing.T) {
tests := []struct {
dbType string
begin string
}{
{dbType: "sqlserver", begin: "BEGIN TRAN"},
{dbType: "sqlserver", begin: "BEGIN DISTRIBUTED TRANSACTION"},
{dbType: "sqlserver", begin: "BEGIN DIALOG CONVERSATION @handle"},
{dbType: "sqlserver", begin: "BEGIN CONVERSATION TIMER (@handle) TIMEOUT = 30"},
{dbType: "sqlite", begin: "BEGIN IMMEDIATE"},
{dbType: "postgres", begin: "BEGIN NOT DEFERRABLE"},
}
for _, test := range tests {
t.Run(test.dbType+" "+test.begin, func(t *testing.T) {
input := test.begin + "; UPDATE demo SET value = 2; COMMIT;"
splitter := &sqlStreamSplitter{dbType: test.dbType}
var got []string
for index := range len(input) {
got = append(got, splitter.Feed([]byte(input[index:index+1]))...)
}
if last := splitter.Flush(); last != "" {
got = append(got, last)
}
want := []string{test.begin, "UPDATE demo SET value = 2", "COMMIT"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("one-byte streaming split of %q = %#v, want %#v", input, got, want)
}
})
}
}
func TestSQLStreamSplitterKeepsMariaDBNotAtomicBlockAcrossChunkBoundaries(t *testing.T) {
block := "BEGIN NOT ATOMIC\n SET @value = 1;\nEND"
input := block + "; SELECT 1;"
splitter := &sqlStreamSplitter{dbType: "mariadb"}
var got []string
for index := range len(input) {
got = append(got, splitter.Feed([]byte(input[index:index+1]))...)
}
if last := splitter.Flush(); last != "" {
got = append(got, last)
}
want := []string{block + ";", "SELECT 1"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("one-byte streaming split of %q = %#v, want %#v", input, got, want)
}
}
func TestSQLStreamSplitterUsesDialectForAmbiguousBeginTran(t *testing.T) {
input := "BEGIN\n TRAN;\nEND;\nSELECT 1 FROM dual;"
splitter := &sqlStreamSplitter{dbType: "oracle"}
var got []string
for index := range len(input) {
got = append(got, splitter.Feed([]byte(input[index:index+1]))...)
}
if last := splitter.Flush(); last != "" {
got = append(got, last)
}
want := []string{"BEGIN\n TRAN;\nEND;", "SELECT 1 FROM dual"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("one-byte Oracle BEGIN TRAN procedure block = %#v, want %#v", got, want)
}
}