feat(sql-audit): 新增 SQL 审计中心并完善事务追踪

- 新增脱敏审计存储、筛选、保留策略、完整性校验与 JSON/CSV 导出
- 覆盖查询编辑器、事务、导入同步、对象操作、AI/MCP 与 Web 运行时入口
- 完善事务完整语句日志、Windows 快捷键映射及 SQL 分析布局
- 补充多语言、健康状态、数据目录迁移和回归测试
This commit is contained in:
Syngnat
2026-07-12 15:24:15 +08:00
parent 0b33b3a683
commit 7c8e8d8dd3
89 changed files with 11421 additions and 194 deletions

235
internal/sqlaudit/export.go Normal file
View File

@@ -0,0 +1,235 @@
package sqlaudit
import (
"bytes"
"context"
"encoding/csv"
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
)
var ErrUnsupportedExportFormat = errors.New("unsupported SQL audit export format")
var (
ErrExportRecordLimit = errors.New("SQL audit export record limit exceeded")
ErrExportSizeLimit = errors.New("SQL audit export size limit exceeded")
)
const (
maxExportRecords int64 = 100_000
maxExportBytes = 64 * 1024 * 1024
)
// BuildExport returns every event matching filter. Pagination fields are
// intentionally ignored so exporting from a paged workbench does not silently
// export only the visible page.
func (s *Store) BuildExport(filter Filter, format string) ([]byte, error) {
return s.buildExportWithLimits(filter, format, maxExportRecords, maxExportBytes)
}
// BuildExportWithLimits applies caller-specific resource caps. It is used by
// the browser server, whose RPC envelope creates additional copies of content.
func (s *Store) BuildExportWithLimits(filter Filter, format string, recordLimit int64, byteLimit int) ([]byte, error) {
if recordLimit <= 0 || recordLimit > maxExportRecords {
recordLimit = maxExportRecords
}
if byteLimit <= 0 || byteLimit > maxExportBytes {
byteLimit = maxExportBytes
}
return s.buildExportWithLimits(filter, format, recordLimit, byteLimit)
}
func (s *Store) buildExportWithLimits(filter Filter, format string, recordLimit int64, byteLimit int) ([]byte, error) {
if err := s.ensureOpen(); err != nil {
return nil, err
}
format = strings.ToLower(strings.TrimSpace(format))
if format != "json" && format != "csv" {
return nil, fmt.Errorf("%w: %s", ErrUnsupportedExportFormat, format)
}
filter = normalizeFilter(filter)
filter.Page = 0
filter.PageSize = 0
events, err := s.loadExportEvents(filter, recordLimit, byteLimit)
if err != nil {
return nil, err
}
switch format {
case "json":
output := newBoundedBuffer(byteLimit)
if _, err := output.Write([]byte{'['}); err != nil {
return nil, err
}
for index, event := range events {
payload, marshalErr := json.Marshal(event)
if marshalErr != nil {
return nil, fmt.Errorf("encode SQL audit JSON export: %w", marshalErr)
}
if index > 0 {
if _, err := output.Write([]byte{','}); err != nil {
return nil, err
}
}
if _, err := output.Write(payload); err != nil {
return nil, err
}
}
if _, err := output.Write([]byte("]\n")); err != nil {
return nil, err
}
return output.Bytes(), nil
case "csv":
output := newBoundedBuffer(byteLimit)
writer := csv.NewWriter(output)
if err := writer.Write(exportCSVHeader); err != nil {
return nil, fmt.Errorf("write SQL audit CSV header: %w", err)
}
for _, event := range events {
if err := writer.Write(eventCSVRecord(event)); err != nil {
return nil, fmt.Errorf("write SQL audit CSV event: %w", err)
}
}
writer.Flush()
if err := writer.Error(); err != nil {
return nil, fmt.Errorf("flush SQL audit CSV export: %w", err)
}
return output.Bytes(), nil
}
return nil, fmt.Errorf("%w: %s", ErrUnsupportedExportFormat, format)
}
func (s *Store) loadExportEvents(filter Filter, recordLimit int64, byteLimit int) ([]Event, error) {
where, args := buildFilterWhere(filter)
var total int64
if err := s.db.QueryRowContext(context.Background(),
`SELECT COUNT(*) FROM sql_audit_events`+where, args...,
).Scan(&total); err != nil {
return nil, fmt.Errorf("count SQL audit export events: %w", err)
}
if recordLimit > 0 && total > recordLimit {
return nil, fmt.Errorf("%w: total=%d limit=%d", ErrExportRecordLimit, total, recordLimit)
}
rows, err := s.db.QueryContext(context.Background(), selectEventColumns+where+
` ORDER BY timestamp DESC, sequence DESC`, args...)
if err != nil {
return nil, fmt.Errorf("query SQL audit export: %w", err)
}
events := make([]Event, 0, int(total))
retainedStringBytes := 0
for rows.Next() {
if recordLimit > 0 && int64(len(events)) >= recordLimit {
_ = rows.Close()
return nil, fmt.Errorf("%w: limit=%d", ErrExportRecordLimit, recordLimit)
}
event, scanErr := scanEvent(rows)
if scanErr != nil {
_ = rows.Close()
return nil, scanErr
}
retainedStringBytes += eventStringBytes(event)
if byteLimit > 0 && retainedStringBytes > byteLimit {
_ = rows.Close()
return nil, fmt.Errorf("%w: limit=%d", ErrExportSizeLimit, byteLimit)
}
events = append(events, event)
}
if err := rows.Err(); err != nil {
_ = rows.Close()
return nil, fmt.Errorf("iterate SQL audit export: %w", err)
}
if err := rows.Close(); err != nil {
return nil, fmt.Errorf("close SQL audit export rows: %w", err)
}
return events, nil
}
func eventStringBytes(event Event) int {
return len(event.ID) + len(event.EventType) + len(event.Status) + len(event.ConnectionID) +
len(event.ConnectionFingerprint) + len(event.DBType) + len(event.Database) + len(event.QueryID) +
len(event.TransactionID) + len(event.Source) + len(event.BoundaryMode) + len(event.CommitMode) +
len(event.SQLText) + len(event.SQLFingerprint) + len(event.Error) + len(event.PrevHash) + len(event.Hash)
}
type boundedBuffer struct {
buffer bytes.Buffer
limit int
}
func newBoundedBuffer(limit int) *boundedBuffer {
return &boundedBuffer{limit: limit}
}
func (b *boundedBuffer) Write(payload []byte) (int, error) {
if b.limit > 0 && b.buffer.Len()+len(payload) > b.limit {
return 0, fmt.Errorf("%w: limit=%d", ErrExportSizeLimit, b.limit)
}
return b.buffer.Write(payload)
}
func (b *boundedBuffer) Bytes() []byte {
return b.buffer.Bytes()
}
var exportCSVHeader = []string{
"id", "sequence", "timestamp", "eventType", "status", "connectionId",
"connectionFingerprint", "dbType", "database", "queryId", "transactionId",
"source", "boundaryMode", "commitMode", "sqlText", "sqlRedacted",
"sqlFingerprint", "statementIndex", "statementCount", "durationMs",
"rowsAffected", "rowsReturned", "error", "prevHash", "hash",
}
func eventCSVRecord(event Event) []string {
return protectCSVRecord([]string{
event.ID,
strconv.FormatInt(event.Sequence, 10),
strconv.FormatInt(event.Timestamp, 10),
event.EventType,
event.Status,
event.ConnectionID,
event.ConnectionFingerprint,
event.DBType,
event.Database,
event.QueryID,
event.TransactionID,
event.Source,
event.BoundaryMode,
event.CommitMode,
event.SQLText,
strconv.FormatBool(event.SQLRedacted),
event.SQLFingerprint,
strconv.Itoa(event.StatementIndex),
strconv.Itoa(event.StatementCount),
strconv.FormatInt(event.DurationMs, 10),
strconv.FormatInt(event.RowsAffected, 10),
strconv.FormatInt(event.RowsReturned, 10),
event.Error,
event.PrevHash,
event.Hash,
})
}
func protectCSVRecord(record []string) []string {
protected := make([]string, len(record))
for index, cell := range record {
protected[index] = protectCSVFormula(cell)
}
return protected
}
func protectCSVFormula(value string) string {
trimmed := strings.TrimLeft(value, " \t\r\n")
if trimmed == "" {
return value
}
switch trimmed[0] {
case '=', '+', '-', '@':
return "'" + value
default:
return value
}
}

View File

@@ -0,0 +1,156 @@
package sqlaudit
import (
"bytes"
"encoding/csv"
"encoding/json"
"errors"
"strings"
"testing"
"time"
)
func TestBuildExportIgnoresPaginationAndFiltersAllMatches(t *testing.T) {
store := openTestStore(t)
now := time.Now().UnixMilli()
for index := 0; index < 3; index++ {
event := sampleEvent("export-"+string(rune('a'+index)), now+int64(index))
event.ConnectionID = "export-connection"
if err := store.Append(event); err != nil {
t.Fatalf("Append returned error: %v", err)
}
}
other := sampleEvent("other", now+10)
other.ConnectionID = "other-connection"
if err := store.Append(other); err != nil {
t.Fatalf("Append other returned error: %v", err)
}
payload, err := store.BuildExport(Filter{
ConnectionID: "export-connection",
Page: 3,
PageSize: 1,
}, "json")
if err != nil {
t.Fatalf("BuildExport(json) returned error: %v", err)
}
var events []Event
if err := json.Unmarshal(payload, &events); err != nil {
t.Fatalf("decode JSON export: %v\npayload=%s", err, payload)
}
if len(events) != 3 {
t.Fatalf("export should ignore pagination and include 3 filtered events, got %d", len(events))
}
for _, event := range events {
if event.ConnectionID != "export-connection" {
t.Fatalf("export ignored non-pagination filter: %#v", event)
}
}
}
func TestBuildCSVExportPreventsFormulaInjection(t *testing.T) {
store := openTestStore(t)
event := sampleEvent("csv-formula", time.Now().UnixMilli())
event.ConnectionID = "=HYPERLINK(\"https://example.test\")"
event.Database = " @SUM(1+1)"
if err := store.Append(event); err != nil {
t.Fatalf("Append returned error: %v", err)
}
payload, err := store.BuildExport(Filter{}, "CSV")
if err != nil {
t.Fatalf("BuildExport(csv) returned error: %v", err)
}
reader := csv.NewReader(bytes.NewReader(payload))
records, err := reader.ReadAll()
if err != nil {
t.Fatalf("read CSV export: %v\npayload=%s", err, payload)
}
if len(records) != 2 {
t.Fatalf("CSV export rows=%d, want header + event", len(records))
}
connectionColumn := csvHeaderIndex(t, records[0], "connectionId")
databaseColumn := csvHeaderIndex(t, records[0], "database")
if !strings.HasPrefix(records[1][connectionColumn], "'=") {
t.Fatalf("connection ID formula was not escaped: %q", records[1][connectionColumn])
}
if !strings.HasPrefix(records[1][databaseColumn], "'@") {
t.Fatalf("whitespace-prefixed formula was not escaped: %q", records[1][databaseColumn])
}
}
func TestBuildExportRejectsUnsupportedFormat(t *testing.T) {
store := openTestStore(t)
_, err := store.BuildExport(Filter{}, "xlsx")
if !errors.Is(err, ErrUnsupportedExportFormat) {
t.Fatalf("BuildExport error=%v, want ErrUnsupportedExportFormat", err)
}
}
func TestBuildExportReturnsRecognizableRecordLimitError(t *testing.T) {
store := openTestStore(t)
now := time.Now().UnixMilli()
if err := store.Append(sampleEvent("limit-a", now)); err != nil {
t.Fatalf("Append first returned error: %v", err)
}
if err := store.Append(sampleEvent("limit-b", now+1)); err != nil {
t.Fatalf("Append second returned error: %v", err)
}
for _, format := range []string{"json", "csv"} {
if _, err := store.buildExportWithLimits(Filter{}, format, 1, maxExportBytes); !errors.Is(err, ErrExportRecordLimit) {
t.Fatalf("%s export error=%v, want ErrExportRecordLimit", format, err)
}
}
}
func TestBuildExportReturnsRecognizableSizeLimitError(t *testing.T) {
store := openTestStore(t)
event := sampleEvent("large-export", time.Now().UnixMilli())
event.Database = strings.Repeat("database", 30)
if err := store.Append(event); err != nil {
t.Fatalf("Append returned error: %v", err)
}
for _, format := range []string{"json", "csv"} {
if _, err := store.buildExportWithLimits(Filter{}, format, maxExportRecords, 64); !errors.Is(err, ErrExportSizeLimit) {
t.Fatalf("%s export error=%v, want ErrExportSizeLimit", format, err)
}
}
}
func TestLoadExportEventsClosesRowsBeforeReturning(t *testing.T) {
store := openTestStore(t)
now := time.Now().UnixMilli()
if err := store.Append(sampleEvent("export-connection-release", now)); err != nil {
t.Fatalf("Append returned error: %v", err)
}
events, err := store.loadExportEvents(normalizeFilter(Filter{}), maxExportRecords, maxExportBytes)
if err != nil || len(events) != 1 {
t.Fatalf("loadExportEvents events=%#v err=%v", events, err)
}
if inUse := store.db.Stats().InUse; inUse != 0 {
t.Fatalf("loadExportEvents returned before releasing SQLite rows: inUse=%d", inUse)
}
done := make(chan error, 1)
go func() {
done <- store.Append(sampleEvent("append-after-export-load", now+1))
}()
select {
case err := <-done:
if err != nil {
t.Fatalf("Append after export load returned error: %v", err)
}
case <-time.After(2 * time.Second):
t.Fatal("export row cursor retained the store's only SQLite connection")
}
}
func csvHeaderIndex(t *testing.T, header []string, name string) int {
t.Helper()
for index, value := range header {
if value == name {
return index
}
}
t.Fatalf("CSV header %q missing from %#v", name, header)
return -1
}

800
internal/sqlaudit/redact.go Normal file
View File

@@ -0,0 +1,800 @@
package sqlaudit
import (
"crypto/sha256"
"encoding/hex"
"regexp"
"strconv"
"strings"
"unicode"
"unicode/utf8"
)
var sqlQueryTypes = map[string]struct{}{
"mysql": {}, "mariadb": {}, "goldendb": {}, "greatdb": {}, "gdb": {},
"oceanbase": {}, "doris": {}, "diros": {}, "starrocks": {}, "sphinx": {},
"postgres": {}, "postgresql": {}, "sqlserver": {}, "mssql": {}, "sqlite": {},
"duckdb": {}, "oracle": {}, "dameng": {}, "dm": {}, "kingbase": {}, "highgo": {},
"vastbase": {}, "opengauss": {}, "gaussdb": {}, "iris": {}, "intersystems": {},
"tdengine": {}, "iotdb": {}, "clickhouse": {}, "trino": {}, "custom": {},
"gonavi": {},
}
var redisReadCommands = map[string]struct{}{
"GET": {}, "MGET": {}, "GETDEL": {}, "GETEX": {}, "STRLEN": {}, "TYPE": {},
"TTL": {}, "PTTL": {}, "EXPIRETIME": {}, "PEXPIRETIME": {}, "EXISTS": {},
"DEL": {}, "UNLINK": {}, "TOUCH": {}, "DUMP": {}, "OBJECT": {}, "KEYS": {},
"SCAN": {}, "HGET": {}, "HMGET": {}, "HGETALL": {}, "HEXISTS": {}, "HLEN": {},
"HKEYS": {}, "HVALS": {}, "HSCAN": {}, "LINDEX": {}, "LLEN": {}, "LRANGE": {},
"LPOS": {}, "SCARD": {}, "SMEMBERS": {}, "SISMEMBER": {}, "SMISMEMBER": {},
"SSCAN": {}, "ZCARD": {}, "ZCOUNT": {}, "ZLEXCOUNT": {}, "ZRANGE": {},
"ZRANGEBYSCORE": {}, "ZRANK": {}, "ZREVRANK": {}, "ZSCORE": {}, "ZMSCORE": {},
"ZSCAN": {}, "XRANGE": {}, "XREVRANGE": {}, "XLEN": {}, "XREAD": {}, "XINFO": {},
"PFCOUNT": {}, "BITCOUNT": {}, "GETBIT": {}, "GEODIST": {}, "GEOHASH": {},
"GEOPOS": {}, "GEOSEARCH": {}, "JSON.GET": {}, "JSON.MGET": {}, "JSON.TYPE": {},
"JSON.OBJKEYS": {}, "JSON.OBJLEN": {}, "JSON.ARRLEN": {}, "TS.GET": {},
"TS.RANGE": {}, "TS.REVRANGE": {}, "TS.MGET": {}, "TS.MRANGE": {},
}
var redisMultiKeyCommands = map[string]struct{}{
"MGET": {}, "DEL": {}, "UNLINK": {}, "EXISTS": {}, "TOUCH": {},
}
// RedactQuery dispatches query sanitization by data-source type. SQL dialects
// use the SQL lexer, Redis retains only commands/keys and replaces values, and
// known non-SQL/message or unknown query languages default to metadata-only.
func RedactQuery(dbType, text string) string {
normalizedType := normalizeQueryType(dbType)
if normalizedType == "redis" {
return redactRedisQuery(text)
}
if _, ok := sqlQueryTypes[normalizedType]; ok {
return RedactSQL(text)
}
return ""
}
const (
maxSQLRunes = 64 * 1024
maxErrorRunes = 4 * 1024
)
var (
sensitiveAssignmentPattern = regexp.MustCompile(`(?i)\b(password|passwd|pwd|token|secret|api[_-]?key|access[_-]?key)\b(\s*(?:=|:)\s*)([^\s,;]+)`)
identifiedByPattern = regexp.MustCompile(`(?i)\b(identified\s+by)(\s+)([^\s,;]+)`)
uriUserInfoPattern = regexp.MustCompile(`(?i)([a-z][a-z0-9+.-]*://)([^\s/@:]+)(?::[^\s/@]*)?@`)
bareUserInfoPattern = regexp.MustCompile(`(?i)\b[^\s:@/]+:[^\s@/]+@(\[[^\]\s]+\]|[a-z0-9.-]+)`)
slashUserInfoPattern = regexp.MustCompile(`(?i)\b[^\s/@:]+/[^\s@/]+@([a-z0-9.-]+)(?:/[^\s,;]*)?`)
bearerPattern = regexp.MustCompile(`(?i)\bbearer\s+[a-z0-9._~+/=-]+`)
basicAuthPattern = regexp.MustCompile(`(?i)\bbasic\s+[a-z0-9+/=]+`)
postgresKeyDetailPattern = regexp.MustCompile(`(?i)(\bkey\s*\([^\)\r\n]*\)\s*=\s*)\([^\)\r\n]*\)`)
duplicateKeyValuePattern = regexp.MustCompile(`(?i)(\bduplicate\s+key\s+value\s+(?:is|was)\s*)\([^\)\r\n]*\)`)
failingRowPattern = regexp.MustCompile(`(?i)(\bfailing\s+row\s+contains\s*)\([^\)\r\n]*\)`)
)
// RedactSQL removes comments and literal values while retaining enough SQL
// structure for local diagnostics. It intentionally favors confidentiality
// over perfectly preserving dialect-specific quoted identifiers.
func RedactSQL(sqlText string) string {
if strings.TrimSpace(sqlText) == "" {
return ""
}
var out strings.Builder
out.Grow(len(sqlText))
for index := 0; index < len(sqlText); {
if hasPrefixAt(sqlText, index, "--") || sqlText[index] == '#' {
index = skipLineComment(sqlText, index)
if index < len(sqlText) && (sqlText[index] == '\r' || sqlText[index] == '\n') {
out.WriteByte('\n')
}
continue
}
if hasPrefixAt(sqlText, index, "/*") {
index = skipBlockComment(sqlText, index+2)
out.WriteByte(' ')
continue
}
if next, ok := skipOracleQuotedLiteral(sqlText, index); ok {
out.WriteByte('?')
index = next
continue
}
if sqlText[index] == '$' {
if next, ok := skipDollarQuotedLiteral(sqlText, index); ok {
out.WriteByte('?')
index = next
continue
}
}
if sqlText[index] == '\'' || sqlText[index] == '"' {
quote := sqlText[index]
out.WriteByte(quote)
out.WriteByte('?')
out.WriteByte(quote)
index = skipQuoted(sqlText, index, quote)
continue
}
if isNumberStart(sqlText, index) {
out.WriteByte('?')
index = skipNumber(sqlText, index)
continue
}
r, size := utf8.DecodeRuneInString(sqlText[index:])
if r == utf8.RuneError && size == 1 {
out.WriteRune(unicode.ReplacementChar)
index++
continue
}
if unicode.IsControl(r) && r != '\n' && r != '\r' && r != '\t' {
out.WriteByte(' ')
} else {
out.WriteRune(r)
}
index += size
}
redacted := sensitiveAssignmentPattern.ReplaceAllString(out.String(), `${1}${2}?`)
redacted = identifiedByPattern.ReplaceAllString(redacted, `${1}${2}?`)
redacted = uriUserInfoPattern.ReplaceAllString(redacted, `${1}***:***@`)
redacted = bearerPattern.ReplaceAllString(redacted, "Bearer ***")
return truncateRunes(strings.TrimSpace(redacted), maxSQLRunes)
}
// RedactError removes common credential forms and quoted values from a driver
// error before it enters the audit database.
func RedactError(message string) string {
message = strings.TrimSpace(message)
if message == "" {
return ""
}
message = uriUserInfoPattern.ReplaceAllString(message, `${1}***:***@`)
message = bareUserInfoPattern.ReplaceAllString(message, `***:***@$1`)
message = slashUserInfoPattern.ReplaceAllString(message, `***/***@$1`)
message = bearerPattern.ReplaceAllString(message, "Bearer ***")
message = basicAuthPattern.ReplaceAllString(message, "Basic ***")
message = postgresKeyDetailPattern.ReplaceAllString(message, `${1}(?)`)
message = duplicateKeyValuePattern.ReplaceAllString(message, `${1}(?)`)
message = failingRowPattern.ReplaceAllString(message, `${1}(?)`)
message = sensitiveAssignmentPattern.ReplaceAllString(message, `${1}${2}***`)
message = identifiedByPattern.ReplaceAllString(message, `${1}${2}***`)
message = redactQuotedSegments(message)
message = strings.Map(func(r rune) rune {
if unicode.IsControl(r) {
return ' '
}
return r
}, message)
return truncateRunes(strings.TrimSpace(message), maxErrorRunes)
}
func normalizeQueryType(dbType string) string {
normalized := strings.ToLower(strings.TrimSpace(dbType))
switch normalized {
case "redis-cluster", "redis_cluster", "redis-sentinel", "redis_sentinel":
return "redis"
case "rocket-mq", "rocket_mq", "apache-rocketmq", "apache_rocketmq", "rmq":
return "rocketmq"
case "apache-kafka", "apache_kafka":
return "kafka"
case "rabbit-mq", "rabbit_mq":
return "rabbitmq"
case "elastic":
return "elasticsearch"
case "chromadb", "chroma-db":
return "chroma"
case "qdrantdb", "qdrant-db":
return "qdrant"
case "milvusdb", "milvus-db":
return "milvus"
default:
return normalized
}
}
func redactRedisQuery(text string) string {
tokens, ok := tokenizeRedisQuery(text)
if !ok || len(tokens) == 0 {
return ""
}
command := strings.ToUpper(tokens[0])
switch command {
case "AUTH":
if len(tokens) >= 3 {
return "AUTH ? ?"
}
if len(tokens) == 2 {
return "AUTH ?"
}
return "AUTH"
case "HELLO":
return redactRedisHello(tokens)
case "CONFIG":
return redactRedisConfig(tokens)
case "ACL":
return redactRedisACL(tokens)
case "CLIENT":
return redactRedisClient(tokens)
case "SCRIPT":
return redactRedisScript(tokens)
case "FUNCTION":
return redactRedisFunction(tokens)
case "PING", "ECHO":
if len(tokens) > 1 {
return command + " ?"
}
return command
case "MSET", "MSETNX":
return redactRedisAlternatingPairs(command, tokens, 1)
case "HSET", "HMSET":
return redactRedisHashPairs(command, tokens)
case "SET", "SETNX", "GETSET", "APPEND", "SETRANGE", "INCRBY", "INCRBYFLOAT",
"DECRBY", "SETBIT", "SETEX", "PSETEX", "RESTORE", "LSET", "LINSERT",
"HSETNX", "HINCRBY", "HINCRBYFLOAT", "LPUSH", "LPUSHX", "RPUSH", "RPUSHX",
"LREM", "SADD", "SREM", "SMOVE",
"ZADD", "ZINCRBY", "ZREM", "GEOADD", "PFADD", "PUBLISH", "SPUBLISH",
"XADD", "XACK", "XCLAIM", "XAUTOCLAIM", "XDEL", "XGROUP", "XTRIM",
"JSON.SET", "JSON.MERGE", "JSON.ARRAPPEND", "JSON.ARRINSERT", "JSON.ARRTRIM",
"JSON.NUMINCRBY", "JSON.NUMMULTBY", "JSON.STRAPPEND", "TS.ADD", "TS.INCRBY",
"TS.DECRBY":
return redactRedisKeyAndPayload(command, tokens)
case "EVAL", "EVALSHA", "FCALL", "FCALL_RO":
return redactRedisScriptInvocation(command, tokens)
case "MIGRATE", "MODULE":
return command + " ?"
case "SELECT", "SWAPDB":
return command + redactRedisSafeArguments(tokens[1:])
default:
if _, ok := redisReadCommands[command]; !ok {
return ""
}
if len(tokens) == 1 {
return command
}
if _, ok := redisMultiKeyCommands[command]; ok {
return command + redactRedisSafeArguments(tokens[1:])
}
// Retain the first key/pattern only. Later arguments can be members,
// payloads or dialect-specific options and are omitted conservatively.
return command + " " + formatRedisToken(tokens[1])
}
}
func redactRedisHello(tokens []string) string {
parts := []string{"HELLO"}
index := 1
if index < len(tokens) {
if _, err := strconv.Atoi(tokens[index]); err == nil {
parts = append(parts, tokens[index])
} else {
parts = append(parts, "?")
}
index++
}
for index < len(tokens) {
switch strings.ToUpper(tokens[index]) {
case "AUTH":
parts = append(parts, "AUTH", "?", "?")
index += 3
case "SETNAME":
parts = append(parts, "SETNAME", "?")
index += 2
default:
// Unknown HELLO options may carry data; retain no further text.
index = len(tokens)
}
}
return strings.Join(parts, " ")
}
func redactRedisConfig(tokens []string) string {
if len(tokens) < 2 {
return "CONFIG"
}
subcommand := strings.ToUpper(tokens[1])
switch subcommand {
case "SET":
if len(tokens) >= 3 {
return "CONFIG SET " + formatRedisToken(tokens[2]) + " ?"
}
return "CONFIG SET ?"
case "GET":
if len(tokens) >= 3 {
return "CONFIG GET " + formatRedisToken(tokens[2])
}
return "CONFIG GET"
case "RESETSTAT", "REWRITE":
return "CONFIG " + subcommand
default:
return "CONFIG"
}
}
func redactRedisACL(tokens []string) string {
if len(tokens) < 2 {
return "ACL"
}
subcommand := strings.ToUpper(tokens[1])
switch subcommand {
case "SETUSER":
if len(tokens) >= 3 {
return "ACL SETUSER " + formatRedisToken(tokens[2]) + " ?"
}
return "ACL SETUSER ?"
case "GETUSER", "DELUSER":
if len(tokens) >= 3 {
return "ACL " + subcommand + " " + formatRedisToken(tokens[2])
}
return "ACL " + subcommand
case "LIST", "USERS", "WHOAMI", "CAT", "GENPASS", "LOG", "SAVE":
return "ACL " + subcommand
default:
return "ACL"
}
}
func redactRedisClient(tokens []string) string {
if len(tokens) < 2 {
return "CLIENT"
}
subcommand := strings.ToUpper(tokens[1])
switch subcommand {
case "SETNAME", "SETINFO", "KILL", "UNBLOCK":
return "CLIENT " + subcommand + " ?"
case "GETNAME", "ID", "INFO", "LIST", "PAUSE", "UNPAUSE", "REPLY", "TRACKING", "CACHING":
return "CLIENT " + subcommand
default:
return "CLIENT"
}
}
func redactRedisScript(tokens []string) string {
if len(tokens) < 2 {
return "SCRIPT"
}
subcommand := strings.ToUpper(tokens[1])
switch subcommand {
case "LOAD", "DEBUG":
return "SCRIPT " + subcommand + " ?"
case "EXISTS", "FLUSH", "KILL", "HELP":
return "SCRIPT " + subcommand
default:
return "SCRIPT"
}
}
func redactRedisFunction(tokens []string) string {
if len(tokens) < 2 {
return "FUNCTION"
}
subcommand := strings.ToUpper(tokens[1])
switch subcommand {
case "LOAD", "RESTORE":
return "FUNCTION " + subcommand + " ?"
case "DELETE":
if len(tokens) >= 3 {
return "FUNCTION DELETE " + formatRedisToken(tokens[2])
}
return "FUNCTION DELETE"
case "DUMP", "FLUSH", "KILL", "LIST", "STATS", "HELP":
return "FUNCTION " + subcommand
default:
return "FUNCTION"
}
}
func redactRedisAlternatingPairs(command string, tokens []string, start int) string {
if len(tokens) <= start || (len(tokens)-start)%2 != 0 {
return command + " ?"
}
parts := []string{command}
for index := start; index < len(tokens); index += 2 {
parts = append(parts, formatRedisToken(tokens[index]))
if index+1 < len(tokens) {
parts = append(parts, "?")
}
}
return strings.Join(parts, " ")
}
func redactRedisHashPairs(command string, tokens []string) string {
if len(tokens) < 2 {
return command
}
if (len(tokens)-2)%2 != 0 {
return command + " " + formatRedisToken(tokens[1]) + " ?"
}
parts := []string{command, formatRedisToken(tokens[1])}
for index := 2; index < len(tokens); index += 2 {
parts = append(parts, formatRedisToken(tokens[index]))
if index+1 < len(tokens) {
parts = append(parts, "?")
}
}
return strings.Join(parts, " ")
}
func redactRedisKeyAndPayload(command string, tokens []string) string {
if len(tokens) < 2 {
return command
}
result := command + " " + formatRedisToken(tokens[1])
if len(tokens) > 2 {
result += " ?"
}
return result
}
func redactRedisScriptInvocation(command string, tokens []string) string {
if len(tokens) < 3 {
return command + " ?"
}
numKeys, err := strconv.Atoi(tokens[2])
if err != nil || numKeys < 0 {
return command + " ?"
}
parts := []string{command, "?", strconv.Itoa(numKeys)}
for index := 0; index < numKeys && 3+index < len(tokens); index++ {
parts = append(parts, formatRedisToken(tokens[3+index]))
}
if 3+numKeys < len(tokens) {
parts = append(parts, "?")
}
return strings.Join(parts, " ")
}
func redactRedisSafeArguments(tokens []string) string {
if len(tokens) == 0 {
return ""
}
parts := make([]string, 0, len(tokens))
for _, token := range tokens {
parts = append(parts, formatRedisToken(token))
}
return " " + strings.Join(parts, " ")
}
func tokenizeRedisQuery(text string) ([]string, bool) {
text = strings.TrimSpace(text)
if text == "" {
return nil, true
}
var tokens []string
var current strings.Builder
var quote rune
escaped := false
tokenStarted := false
flush := func() {
if !tokenStarted {
return
}
tokens = append(tokens, current.String())
current.Reset()
tokenStarted = false
}
for _, r := range text {
if escaped {
current.WriteRune(r)
escaped = false
tokenStarted = true
continue
}
if r == '\\' {
escaped = true
tokenStarted = true
continue
}
if quote != 0 {
if r == quote {
quote = 0
} else {
current.WriteRune(r)
}
tokenStarted = true
continue
}
if r == '\'' || r == '"' {
quote = r
tokenStarted = true
continue
}
if unicode.IsSpace(r) {
flush()
continue
}
current.WriteRune(r)
tokenStarted = true
}
if escaped {
current.WriteRune('\\')
}
if quote != 0 {
return nil, false
}
flush()
return tokens, true
}
func formatRedisToken(token string) string {
token = truncateRunes(strings.Map(func(r rune) rune {
if unicode.IsControl(r) {
return ' '
}
return r
}, token), 512)
if token == "" {
return `""`
}
if strings.IndexFunc(token, unicode.IsSpace) >= 0 {
return strconv.Quote(token)
}
return token
}
func queryFingerprintSeed(dbType, redactedText string) string {
if redactedText != "" {
return redactedText
}
// Metadata-only query languages intentionally do not include raw input in
// their fingerprint. Payload-derived hashes can be dictionary-guessed when
// messages contain low-entropy tokens, card numbers or common credentials.
return "metadata:" + normalizeQueryType(dbType)
}
func sanitizeEvent(event Event, settings Settings) Event {
event.Sequence = 0
event.PrevHash = ""
event.Hash = ""
event.EventType = strings.ToLower(sanitizeLabel(event.EventType, 64))
event.Status = strings.ToLower(sanitizeLabel(event.Status, 64))
event.ConnectionID = sanitizeLabel(event.ConnectionID, 256)
event.DBType = strings.ToLower(sanitizeLabel(event.DBType, 64))
event.Database = sanitizeLabel(event.Database, 512)
event.QueryID = sanitizeLabel(event.QueryID, 256)
event.TransactionID = sanitizeLabel(event.TransactionID, 256)
event.Source = strings.ToLower(sanitizeLabel(event.Source, 64))
event.BoundaryMode = normalizeBoundaryMode(event.BoundaryMode)
event.CommitMode = normalizeCommitMode(event.CommitMode)
rawSQL := event.SQLText
redactedSQL := RedactQuery(event.DBType, rawSQL)
fingerprintSeed := queryFingerprintSeed(event.DBType, redactedSQL)
event.SQLFingerprint = versionedDigest("sqlaudit-sql-fingerprint-v1", fingerprintSeed)
event.SQLRedacted = true
if settings.CaptureMode == CaptureModeMetadata {
event.SQLText = ""
} else {
event.SQLText = redactedSQL
}
fingerprintSeed = strings.TrimSpace(event.ConnectionFingerprint)
if fingerprintSeed == "" {
fingerprintSeed = strings.Join([]string{event.ConnectionID, event.DBType, event.Database}, "\x00")
}
event.ConnectionFingerprint = versionedDigest("sqlaudit-connection-fingerprint-v1", fingerprintSeed)
event.Error = RedactError(event.Error)
if event.StatementIndex < 0 {
event.StatementIndex = 0
}
if event.StatementCount < 0 {
event.StatementCount = 0
}
if event.DurationMs < 0 {
event.DurationMs = 0
}
if event.RowsAffected < 0 {
event.RowsAffected = 0
}
if event.RowsReturned < 0 {
event.RowsReturned = 0
}
return event
}
func normalizeBoundaryMode(value string) string {
switch strings.ToLower(strings.TrimSpace(value)) {
case BoundaryModeDriverAPI:
return BoundaryModeDriverAPI
case BoundaryModeTextSQL:
return BoundaryModeTextSQL
case BoundaryModeImplicit:
return BoundaryModeImplicit
default:
return BoundaryModeUnknown
}
}
func normalizeCommitMode(value string) string {
switch strings.ToLower(strings.TrimSpace(value)) {
case CommitModeAuto:
return CommitModeAuto
case CommitModeManual:
return CommitModeManual
case CommitModePending:
return CommitModePending
default:
return ""
}
}
func versionedDigest(version, value string) string {
digest := sha256.Sum256([]byte(version + "\x00" + value))
return hex.EncodeToString(digest[:])
}
func sanitizeLabel(value string, maxRunes int) string {
value = strings.Map(func(r rune) rune {
if unicode.IsControl(r) {
return ' '
}
return r
}, strings.TrimSpace(value))
return truncateRunes(value, maxRunes)
}
func truncateRunes(value string, max int) string {
if max <= 0 || utf8.RuneCountInString(value) <= max {
return value
}
runes := []rune(value)
return string(runes[:max])
}
func redactQuotedSegments(value string) string {
var out strings.Builder
out.Grow(len(value))
for index := 0; index < len(value); {
if value[index] == '\'' || value[index] == '"' {
quote := value[index]
out.WriteByte(quote)
out.WriteByte('?')
out.WriteByte(quote)
index = skipQuoted(value, index, quote)
continue
}
out.WriteByte(value[index])
index++
}
return out.String()
}
func skipQuoted(value string, start int, quote byte) int {
for index := start + 1; index < len(value); index++ {
if value[index] == '\\' {
index++
continue
}
if value[index] != quote {
continue
}
if index+1 < len(value) && value[index+1] == quote {
index++
continue
}
return index + 1
}
return len(value)
}
func skipLineComment(value string, start int) int {
for index := start; index < len(value); index++ {
if value[index] == '\r' || value[index] == '\n' {
return index
}
}
return len(value)
}
func skipBlockComment(value string, start int) int {
depth := 1
for index := start; index+1 < len(value); {
switch value[index : index+2] {
case "/*":
depth++
index += 2
case "*/":
depth--
index += 2
if depth == 0 {
return index
}
default:
index++
}
}
return len(value)
}
func skipDollarQuotedLiteral(value string, start int) (int, bool) {
endTag := start + 1
for endTag < len(value) && (value[endTag] == '_' || isASCIILetterOrDigit(value[endTag])) {
endTag++
}
if endTag >= len(value) || value[endTag] != '$' {
return start, false
}
tag := value[start : endTag+1]
contentEnd := strings.Index(value[endTag+1:], tag)
if contentEnd < 0 {
return len(value), true
}
return endTag + 1 + contentEnd + len(tag), true
}
func skipOracleQuotedLiteral(value string, start int) (int, bool) {
if start+3 >= len(value) || (value[start] != 'q' && value[start] != 'Q') || value[start+1] != '\'' {
return start, false
}
opening := value[start+2]
closing := opening
switch opening {
case '[':
closing = ']'
case '{':
closing = '}'
case '(':
closing = ')'
case '<':
closing = '>'
}
for index := start + 3; index+1 < len(value); index++ {
if value[index] == closing && value[index+1] == '\'' {
return index + 2, true
}
}
return len(value), true
}
func isNumberStart(value string, index int) bool {
if index >= len(value) || value[index] < '0' || value[index] > '9' {
return false
}
if index == 0 {
return true
}
previous := value[index-1]
return !(previous == '_' || previous == '$' || isASCIILetterOrDigit(previous))
}
func skipNumber(value string, start int) int {
index := start
if index+2 <= len(value) && index+1 < len(value) && value[index] == '0' && (value[index+1] == 'x' || value[index+1] == 'X') {
index += 2
for index < len(value) && isASCIIHex(value[index]) {
index++
}
return index
}
for index < len(value) && value[index] >= '0' && value[index] <= '9' {
index++
}
if index < len(value) && value[index] == '.' {
index++
for index < len(value) && value[index] >= '0' && value[index] <= '9' {
index++
}
}
if index < len(value) && (value[index] == 'e' || value[index] == 'E') {
index++
if index < len(value) && (value[index] == '+' || value[index] == '-') {
index++
}
for index < len(value) && value[index] >= '0' && value[index] <= '9' {
index++
}
}
return index
}
func hasPrefixAt(value string, index int, prefix string) bool {
return index >= 0 && index+len(prefix) <= len(value) && value[index:index+len(prefix)] == prefix
}
func isASCIILetterOrDigit(value byte) bool {
return value >= 'a' && value <= 'z' || value >= 'A' && value <= 'Z' || value >= '0' && value <= '9'
}
func isASCIIHex(value byte) bool {
return value >= '0' && value <= '9' || value >= 'a' && value <= 'f' || value >= 'A' && value <= 'F'
}

View File

@@ -0,0 +1,167 @@
package sqlaudit
import (
"strings"
"testing"
)
func TestRedactSQLRemovesDialectLiteralsCommentsAndCredentials(t *testing.T) {
input := `-- comment-secret
SELECT 'alice-secret', 12345, $$postgres-secret$$, q'[oracle-secret]'
FROM users /* block-secret */
WHERE password = bare-secret
AND endpoint = 'postgres://alice:db-secret@example.test/app'
AND note = "quoted-secret"`
redacted := RedactSQL(input)
for _, secret := range []string{
"comment-secret", "alice-secret", "12345", "postgres-secret", "oracle-secret",
"block-secret", "bare-secret", "alice", "db-secret", "quoted-secret",
} {
if strings.Contains(redacted, secret) {
t.Fatalf("redacted SQL leaked %q: %s", secret, redacted)
}
}
if !strings.Contains(redacted, "SELECT") || !strings.Contains(redacted, "FROM users") {
t.Fatalf("redaction should retain SQL structure: %s", redacted)
}
if !strings.Contains(redacted, "password = ?") {
t.Fatalf("sensitive unquoted assignment should be replaced: %s", redacted)
}
}
func TestRedactSQLPreservesIdentifiersAndPlaceholders(t *testing.T) {
input := "SELECT report_2026, `OrderID`, [AccountID] FROM events WHERE id = $1 AND name = ?"
redacted := RedactSQL(input)
for _, fragment := range []string{"report_2026", "`OrderID`", "[AccountID]", "$1", "name = ?"} {
if !strings.Contains(redacted, fragment) {
t.Fatalf("expected %q to remain in %q", fragment, redacted)
}
}
}
func TestRedactErrorRemovesSecretsAndControlCharacters(t *testing.T) {
input := "connect postgres://alice:db-secret@example.test/app password=top-secret " +
"Authorization: Bearer abc.def.ghi Basic dXNlcjpwYXNz fallback alice:bare-pass@db.internal oracle scott/tiger@dbhost/service near 'literal-secret'\nnext"
redacted := RedactError(input)
for _, secret := range []string{"alice", "db-secret", "top-secret", "abc.def.ghi", "dXNlcjpwYXNz", "bare-pass", "scott", "tiger", "service", "literal-secret"} {
if strings.Contains(redacted, secret) {
t.Fatalf("redacted error leaked %q: %s", secret, redacted)
}
}
if strings.ContainsAny(redacted, "\r\n") {
t.Fatalf("redacted error should be single line: %q", redacted)
}
}
func TestRedactSQLRemovesNestedBlockComments(t *testing.T) {
redacted := RedactSQL("SELECT 1 /* outer /* nested */ private-after-nested */ FROM users")
if strings.Contains(redacted, "outer") || strings.Contains(redacted, "nested") || strings.Contains(redacted, "private-after-nested") {
t.Fatalf("nested SQL comment leaked into audit text: %q", redacted)
}
if !strings.Contains(redacted, "FROM users") {
t.Fatalf("nested comment redaction removed following SQL structure: %q", redacted)
}
}
func TestRedactSQLPreservesSensitiveNamedColumnsWithoutAssignments(t *testing.T) {
input := "SELECT password, token, secret FROM sessions"
if redacted := RedactSQL(input); redacted != input {
t.Fatalf("sensitive-named columns were mistaken for assigned values: %q", redacted)
}
}
func TestRedactErrorRemovesUnquotedDuplicateKeyValues(t *testing.T) {
input := "ERROR: duplicate key value violates unique constraint; DETAIL: Key (email, tenant_id)=(alice@example.test, 42) already exists; The duplicate key value is (private-token). Failing row contains (7, failing@example.test, row-secret)."
redacted := RedactError(input)
for _, secret := range []string{"alice@example.test", "42", "private-token", "failing@example.test", "row-secret"} {
if strings.Contains(redacted, secret) {
t.Fatalf("redacted duplicate-key error leaked %q: %s", secret, redacted)
}
}
if !strings.Contains(redacted, "Key (email, tenant_id)=(?)") {
t.Fatalf("redacted error lost safe constraint structure: %s", redacted)
}
}
func TestRedactionBoundsStoredText(t *testing.T) {
longSQL := "SELECT '" + strings.Repeat("界", maxSQLRunes+100) + "'"
if got := len([]rune(RedactSQL(longSQL))); got > maxSQLRunes {
t.Fatalf("redacted SQL exceeds limit: %d", got)
}
longError := strings.Repeat("failure ", maxErrorRunes)
if got := len([]rune(RedactError(longError))); got > maxErrorRunes {
t.Fatalf("redacted error exceeds limit: %d", got)
}
}
func TestRedactQueryUsesSQLLexerForSQLTypes(t *testing.T) {
query := "SELECT * FROM users WHERE password = 'sql-secret' AND id = 42"
for _, dbType := range []string{"mysql", "postgres", "sqlserver", "oracle", "clickhouse", "custom"} {
if got, want := RedactQuery(dbType, query), RedactSQL(query); got != want {
t.Fatalf("RedactQuery(%s)=%q, want SQL redaction %q", dbType, got, want)
}
}
}
func TestRedactQueryRedactsRedisValuesAndPreservesCommandsAndKeys(t *testing.T) {
tests := []struct {
name string
query string
want string
secrets []string
}{
{name: "auth", query: "AUTH default auth-secret", want: "AUTH ? ?", secrets: []string{"default", "auth-secret"}},
{name: "hello auth", query: "HELLO 3 AUTH app hello-secret SETNAME client-secret", want: "HELLO 3 AUTH ? ? SETNAME ?", secrets: []string{"app", "hello-secret", "client-secret"}},
{name: "config set", query: "CONFIG SET requirepass config-secret", want: "CONFIG SET requirepass ?", secrets: []string{"config-secret"}},
{name: "set", query: "SET session:key set-secret EX 60", want: "SET session:key ?", secrets: []string{"set-secret", "60"}},
{name: "mset", query: "MSET key:1 first-secret key:2 second-secret", want: "MSET key:1 ? key:2 ?", secrets: []string{"first-secret", "second-secret"}},
{name: "hset", query: "HSET profile:1 email private@example.test token hash-secret", want: "HSET profile:1 email ? token ?", secrets: []string{"private@example.test", "hash-secret"}},
{name: "list", query: "LPUSH queue:key payload-one payload-two", want: "LPUSH queue:key ?", secrets: []string{"payload-one", "payload-two"}},
{name: "set member", query: "SADD set:key member-secret", want: "SADD set:key ?", secrets: []string{"member-secret"}},
{name: "sorted set", query: "ZADD rank:key 12.5 member-secret", want: "ZADD rank:key ?", secrets: []string{"12.5", "member-secret"}},
{name: "publish", query: "PUBLISH channel:key message-secret", want: "PUBLISH channel:key ?", secrets: []string{"message-secret"}},
{name: "stream", query: "XADD stream:key * field payload-secret", want: "XADD stream:key ?", secrets: []string{"payload-secret", "field"}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := RedactQuery("redis", tt.query)
if got != tt.want {
t.Fatalf("RedactQuery(redis)=%q, want %q", got, tt.want)
}
for _, secret := range tt.secrets {
if strings.Contains(got, secret) {
t.Fatalf("Redis redaction leaked %q in %q", secret, got)
}
}
})
}
}
func TestRedactQueryDefaultsMessageAndUnknownLanguagesToMetadataOnly(t *testing.T) {
query := `PUBLISH topic {"token":"message-secret"}`
for _, dbType := range []string{"rocketmq", "mqtt", "kafka", "rabbitmq", "mongodb", "elasticsearch", "milvus", "mystery-driver"} {
if got := RedactQuery(dbType, query); got != "" {
t.Fatalf("RedactQuery(%s)=%q, want metadata-only empty text", dbType, got)
}
}
if got := RedactQuery("redis", `SET key "unterminated-secret`); got != "" {
t.Fatalf("unparseable Redis input must fall back to metadata-only, got %q", got)
}
}
func TestSanitizeEventPreservesPendingCommitMode(t *testing.T) {
event := sanitizeEvent(Event{
EventType: "transaction_begin",
Status: "success",
CommitMode: CommitModePending,
}, Settings{
Enabled: true,
CaptureMode: CaptureModeRedacted,
RetentionDays: 30,
MaxRecords: 100,
})
if event.CommitMode != CommitModePending {
t.Fatalf("pending commit mode = %q, want %q", event.CommitMode, CommitModePending)
}
}

880
internal/sqlaudit/store.go Normal file
View File

@@ -0,0 +1,880 @@
package sqlaudit
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"net/url"
"os"
"path/filepath"
"runtime"
"strings"
"time"
"github.com/google/uuid"
_ "modernc.org/sqlite"
)
const (
schemaVersion = 1
chainVersion = "sqlaudit-chain-v1"
integrityAlgorithm = "sha256-canonical-v1"
maxRetentionDays = 3650
maxConfiguredEvents = 10_000_000
)
var (
ErrClosed = errors.New("sql audit store is closed")
ErrBatchExceedsMaxRecords = errors.New("sql audit batch exceeds configured max records")
)
type Store struct {
db *sql.DB
path string
}
func Open(path string) (*Store, error) {
path = strings.TrimSpace(path)
if path == "" {
return nil, errors.New("sql audit database path is empty")
}
absPath, err := filepath.Abs(path)
if err != nil {
return nil, fmt.Errorf("resolve sql audit database path: %w", err)
}
dir := filepath.Dir(absPath)
if err := os.MkdirAll(dir, 0o700); err != nil {
return nil, fmt.Errorf("create sql audit directory: %w", err)
}
if err := os.Chmod(dir, 0o700); err != nil {
return nil, fmt.Errorf("secure sql audit directory: %w", err)
}
database, err := sql.Open("sqlite", sqliteAuditDSN(absPath))
if err != nil {
return nil, fmt.Errorf("open sql audit database: %w", err)
}
database.SetMaxOpenConns(4)
database.SetMaxIdleConns(4)
store := &Store{db: database, path: absPath}
if err := store.initialize(); err != nil {
_ = database.Close()
return nil, err
}
if err := os.Chmod(absPath, 0o600); err != nil {
_ = database.Close()
return nil, fmt.Errorf("secure sql audit database: %w", err)
}
return store, nil
}
func (s *Store) Path() string {
if s == nil {
return ""
}
return s.path
}
func (s *Store) Close() error {
if s == nil || s.db == nil {
return nil
}
// Consolidate WAL content before a data-root migration or application exit.
// App lifecycle locking ensures no new store operation starts during Close.
_, checkpointErr := s.db.ExecContext(context.Background(), `PRAGMA wal_checkpoint(TRUNCATE)`)
err := errors.Join(checkpointErr, s.db.Close())
s.db = nil
return err
}
func sqliteAuditDSN(path string) string {
uriPath := filepath.ToSlash(path)
dsn := &url.URL{Scheme: "file", Path: uriPath}
if runtime.GOOS == "windows" {
if strings.HasPrefix(uriPath, "//") {
withoutPrefix := strings.TrimPrefix(uriPath, "//")
if separator := strings.IndexByte(withoutPrefix, '/'); separator >= 0 {
dsn.Host = withoutPrefix[:separator]
dsn.Path = withoutPrefix[separator:]
}
} else if !strings.HasPrefix(uriPath, "/") {
dsn.Path = "/" + uriPath
}
}
query := url.Values{}
for _, pragma := range []string{
"busy_timeout(5000)",
"foreign_keys(ON)",
"synchronous(FULL)",
"journal_mode(WAL)",
} {
query.Add("_pragma", pragma)
}
dsn.RawQuery = query.Encode()
return dsn.String()
}
func (s *Store) initialize() error {
if err := s.ensureOpen(); err != nil {
return err
}
ctx := context.Background()
for _, pragma := range []string{
"PRAGMA busy_timeout=5000",
"PRAGMA journal_mode=WAL",
"PRAGMA synchronous=FULL",
"PRAGMA foreign_keys=ON",
} {
if _, err := s.db.ExecContext(ctx, pragma); err != nil {
return fmt.Errorf("configure sql audit database (%s): %w", pragma, err)
}
}
var existingSchemaVersion int
if err := s.db.QueryRowContext(ctx, `PRAGMA user_version`).Scan(&existingSchemaVersion); err != nil {
return fmt.Errorf("read SQL audit schema version: %w", err)
}
if existingSchemaVersion > schemaVersion {
return fmt.Errorf("SQL audit schema version %d is newer than supported version %d", existingSchemaVersion, schemaVersion)
}
statements := []string{
`CREATE TABLE IF NOT EXISTS sql_audit_events (
sequence INTEGER PRIMARY KEY AUTOINCREMENT,
id TEXT NOT NULL UNIQUE,
timestamp INTEGER NOT NULL,
event_type TEXT NOT NULL,
status TEXT NOT NULL,
connection_id TEXT NOT NULL DEFAULT '',
connection_fingerprint TEXT NOT NULL,
db_type TEXT NOT NULL DEFAULT '',
database_name TEXT NOT NULL DEFAULT '',
query_id TEXT NOT NULL DEFAULT '',
transaction_id TEXT NOT NULL DEFAULT '',
source TEXT NOT NULL DEFAULT '',
boundary_mode TEXT NOT NULL DEFAULT 'unknown',
commit_mode TEXT NOT NULL DEFAULT '',
sql_text TEXT NOT NULL DEFAULT '',
sql_redacted INTEGER NOT NULL DEFAULT 1,
sql_fingerprint TEXT NOT NULL,
statement_index INTEGER NOT NULL DEFAULT 0,
statement_count INTEGER NOT NULL DEFAULT 0,
duration_ms INTEGER NOT NULL DEFAULT 0,
rows_affected INTEGER NOT NULL DEFAULT 0,
rows_returned INTEGER NOT NULL DEFAULT 0,
error_text TEXT NOT NULL DEFAULT '',
prev_hash TEXT NOT NULL DEFAULT '',
record_hash TEXT NOT NULL
)`,
`CREATE INDEX IF NOT EXISTS idx_sql_audit_events_timestamp ON sql_audit_events(timestamp DESC, sequence DESC)`,
`CREATE INDEX IF NOT EXISTS idx_sql_audit_events_connection ON sql_audit_events(connection_id, timestamp DESC)`,
`CREATE INDEX IF NOT EXISTS idx_sql_audit_events_transaction ON sql_audit_events(transaction_id, sequence)`,
`CREATE INDEX IF NOT EXISTS idx_sql_audit_events_type_status ON sql_audit_events(event_type, status, timestamp DESC)`,
`CREATE TABLE IF NOT EXISTS sql_audit_settings (
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
enabled INTEGER NOT NULL,
capture_mode TEXT NOT NULL,
retention_days INTEGER NOT NULL,
max_records INTEGER NOT NULL
)`,
`INSERT OR IGNORE INTO sql_audit_settings(singleton, enabled, capture_mode, retention_days, max_records)
VALUES(1, 1, 'redacted', 30, 100000)`,
fmt.Sprintf("PRAGMA user_version=%d", schemaVersion),
}
for _, statement := range statements {
if _, err := s.db.ExecContext(ctx, statement); err != nil {
return fmt.Errorf("initialize sql audit database: %w", err)
}
}
return nil
}
// Append adds one terminal audit fact. When auditing is disabled this method is
// a successful no-op. SQL and error text are sanitized before the transaction.
func (s *Store) Append(event Event) error {
return s.AppendBatch([]Event{event})
}
// AppendControl persists an audit-control event even when ordinary collection
// is disabled. It is reserved for settings and purge boundaries so disabling
// or clearing the audit stream cannot create an unmarked control-plane window.
func (s *Store) AppendControl(event Event) error {
return s.appendBatch([]Event{event}, true)
}
// AppendBatch persists related facts in one durable SQLite transaction. It is
// primarily used for the statements executed inside one managed transaction,
// avoiding one FULL-sync round trip per statement while keeping the complete
// batch visible before the database transaction is returned to the caller.
func (s *Store) AppendBatch(events []Event) error {
return s.appendBatch(events, false)
}
func (s *Store) appendBatch(events []Event, bypassDisabled bool) error {
if err := s.ensureOpen(); err != nil {
return err
}
if len(events) == 0 {
return nil
}
inputEvents := events
return s.withImmediate(func(conn *sql.Conn) error {
settings, err := getSettingsFrom(conn)
if err != nil {
return err
}
if !settings.Enabled && !bypassDisabled {
return nil
}
return appendEventsLocked(conn, settings, inputEvents)
})
}
func appendEventsLocked(conn *sql.Conn, settings Settings, inputEvents []Event) error {
if len(inputEvents) > settings.MaxRecords {
return fmt.Errorf(
"%w: batch=%d limit=%d",
ErrBatchExceedsMaxRecords,
len(inputEvents),
settings.MaxRecords,
)
}
prepared := append([]Event(nil), inputEvents...)
now := time.Now().UnixMilli()
for index := range prepared {
event := &prepared[index]
if event.Timestamp <= 0 {
event.Timestamp = now
}
if strings.TrimSpace(event.ID) == "" {
event.ID = uuid.NewString()
} else {
event.ID = sanitizeLabel(event.ID, 256)
}
if strings.TrimSpace(event.EventType) == "" {
return fmt.Errorf("sql audit event %d type is required", index+1)
}
if strings.TrimSpace(event.Status) == "" {
return fmt.Errorf("sql audit event %d status is required", index+1)
}
}
if _, err := pruneLocked(conn, settings, time.Now(), int64(len(prepared))); err != nil {
return err
}
var previousHash string
err := conn.QueryRowContext(context.Background(),
`SELECT record_hash FROM sql_audit_events ORDER BY sequence DESC LIMIT 1`,
).Scan(&previousHash)
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return fmt.Errorf("read sql audit chain head: %w", err)
}
for index := range prepared {
event := sanitizeEvent(prepared[index], settings)
event.PrevHash = previousHash
result, err := conn.ExecContext(context.Background(), `INSERT INTO sql_audit_events(
id, timestamp, event_type, status, connection_id, connection_fingerprint,
db_type, database_name, query_id, transaction_id, source, boundary_mode,
commit_mode, sql_text, sql_redacted, sql_fingerprint, statement_index,
statement_count, duration_ms, rows_affected, rows_returned, error_text,
prev_hash, record_hash
) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
event.ID, event.Timestamp, event.EventType, event.Status, event.ConnectionID,
event.ConnectionFingerprint, event.DBType, event.Database, event.QueryID,
event.TransactionID, event.Source, event.BoundaryMode, event.CommitMode,
event.SQLText, boolToInt(event.SQLRedacted), event.SQLFingerprint,
event.StatementIndex, event.StatementCount, event.DurationMs,
event.RowsAffected, event.RowsReturned, event.Error, event.PrevHash, "",
)
if err != nil {
return fmt.Errorf("append sql audit event %d: %w", index+1, err)
}
event.Sequence, err = result.LastInsertId()
if err != nil {
return fmt.Errorf("read sql audit sequence for event %d: %w", index+1, err)
}
event.Hash, err = calculateEventHash(event)
if err != nil {
return err
}
if _, err := conn.ExecContext(context.Background(),
`UPDATE sql_audit_events SET record_hash=? WHERE sequence=?`, event.Hash, event.Sequence,
); err != nil {
return fmt.Errorf("finalize sql audit event %d hash: %w", index+1, err)
}
previousHash = event.Hash
}
return nil
}
func (s *Store) Query(filter Filter) (Page, error) {
if err := s.ensureOpen(); err != nil {
return Page{}, err
}
filter = normalizeFilter(filter)
where, args := buildFilterWhere(filter)
page := Page{Items: []Event{}, Page: filter.Page, PageSize: filter.PageSize}
summarySQL := `SELECT COUNT(*),
COALESCE(SUM(CASE WHEN status='success' THEN 1 ELSE 0 END), 0),
COALESCE(SUM(CASE WHEN status='error' THEN 1 ELSE 0 END), 0),
COUNT(DISTINCT CASE WHEN transaction_id <> '' THEN transaction_id END)
FROM sql_audit_events` + where
if err := s.db.QueryRowContext(context.Background(), summarySQL, args...).Scan(
&page.Summary.TotalEvents,
&page.Summary.SuccessCount,
&page.Summary.ErrorCount,
&page.Summary.TransactionCount,
); err != nil {
return Page{}, fmt.Errorf("summarize sql audit events: %w", err)
}
page.Total = page.Summary.TotalEvents
queryArgs := append(append([]any{}, args...), filter.PageSize, (filter.Page-1)*filter.PageSize)
rows, err := s.db.QueryContext(context.Background(), selectEventColumns+where+
` ORDER BY timestamp DESC, sequence DESC LIMIT ? OFFSET ?`, queryArgs...)
if err != nil {
return Page{}, fmt.Errorf("query sql audit events: %w", err)
}
defer rows.Close()
for rows.Next() {
event, err := scanEvent(rows)
if err != nil {
return Page{}, err
}
page.Items = append(page.Items, event)
}
if err := rows.Err(); err != nil {
return Page{}, fmt.Errorf("iterate sql audit events: %w", err)
}
return page, nil
}
func (s *Store) UpdateSettings(settings Settings) error {
if err := s.ensureOpen(); err != nil {
return err
}
normalized, err := normalizeSettings(settings)
if err != nil {
return err
}
return s.withImmediate(func(conn *sql.Conn) error {
if _, err := conn.ExecContext(context.Background(), `UPDATE sql_audit_settings
SET enabled=?, capture_mode=?, retention_days=?, max_records=? WHERE singleton=1`,
boolToInt(normalized.Enabled), normalized.CaptureMode,
normalized.RetentionDays, normalized.MaxRecords,
); err != nil {
return fmt.Errorf("update sql audit settings: %w", err)
}
_, err := pruneLocked(conn, normalized, time.Now(), 0)
return err
})
}
// UpdateSettingsWithControl atomically applies settings and appends a
// non-disableable control boundary describing the old and new safe values.
func (s *Store) UpdateSettingsWithControl(settings Settings, control Event) error {
if err := s.ensureOpen(); err != nil {
return err
}
normalized, err := normalizeSettings(settings)
if err != nil {
return err
}
return s.withImmediate(func(conn *sql.Conn) error {
previous, err := getSettingsFrom(conn)
if err != nil {
return err
}
if _, err := conn.ExecContext(context.Background(), `UPDATE sql_audit_settings
SET enabled=?, capture_mode=?, retention_days=?, max_records=? WHERE singleton=1`,
boolToInt(normalized.Enabled), normalized.CaptureMode,
normalized.RetentionDays, normalized.MaxRecords,
); err != nil {
return fmt.Errorf("update sql audit settings: %w", err)
}
control.DBType = "gonavi"
control.SQLText = settingsControlDescriptor(previous, normalized)
return appendEventsLocked(conn, controlAuditSettings(normalized), []Event{control})
})
}
// Control descriptors contain only GoNavi-owned structural metadata. Keep
// them visible even when ordinary events use metadata-only capture, otherwise
// disable/retention/purge boundaries would lose the values needed to explain
// an audit window.
func controlAuditSettings(settings Settings) Settings {
settings.CaptureMode = CaptureModeRedacted
return settings
}
func settingsControlDescriptor(previous, next Settings) string {
return fmt.Sprintf(
"AUDIT SETTINGS FROM_ENABLED_%s TO_ENABLED_%s FROM_CAPTURE_%s TO_CAPTURE_%s FROM_RETENTION_DAYS_%d TO_RETENTION_DAYS_%d FROM_MAX_RECORDS_%d TO_MAX_RECORDS_%d",
settingsBoolToken(previous.Enabled),
settingsBoolToken(next.Enabled),
strings.ToUpper(previous.CaptureMode),
strings.ToUpper(next.CaptureMode),
previous.RetentionDays,
next.RetentionDays,
previous.MaxRecords,
next.MaxRecords,
)
}
func settingsBoolToken(value bool) string {
if value {
return "ON"
}
return "OFF"
}
func (s *Store) GetSettings() (Settings, error) {
if err := s.ensureOpen(); err != nil {
return Settings{}, err
}
return getSettingsFrom(s.db)
}
// Clear deletes the oldest contiguous sequence prefix whose timestamps are
// older than beforeTimestamp. A non-positive timestamp clears all events.
// Remaining records and their hashes are never rewritten; integrity checks
// report a truncated/partial chain when a retained first record still points
// at a deleted predecessor.
func (s *Store) Clear(beforeTimestamp int64) (int64, error) {
if err := s.ensureOpen(); err != nil {
return 0, err
}
var deleted int64
err := s.withImmediate(func(conn *sql.Conn) error {
var err error
if beforeTimestamp <= 0 {
deleted, err = deleteAllLocked(conn)
} else {
deleted, err = deleteTimestampPrefixLocked(conn, beforeTimestamp)
}
return err
})
return deleted, err
}
// ClearWithControl atomically removes the selected prefix and appends a control
// boundary that remains visible even when ordinary collection is disabled.
func (s *Store) ClearWithControl(beforeTimestamp int64, control Event) (int64, error) {
if err := s.ensureOpen(); err != nil {
return 0, err
}
var deleted int64
err := s.withImmediate(func(conn *sql.Conn) error {
settings, err := getSettingsFrom(conn)
if err != nil {
return err
}
if beforeTimestamp <= 0 {
deleted, err = deleteAllLocked(conn)
control.SQLText = "AUDIT CLEAR ALL"
} else {
deleted, err = deleteTimestampPrefixLocked(conn, beforeTimestamp)
control.SQLText = fmt.Sprintf("AUDIT CLEAR BEFORE_TIMESTAMP_%d", beforeTimestamp)
}
if err != nil {
return err
}
control.DBType = "gonavi"
control.RowsAffected = deleted
return appendEventsLocked(conn, controlAuditSettings(settings), []Event{control})
})
return deleted, err
}
func (s *Store) VerifyIntegrity() (IntegrityReport, error) {
report := IntegrityReport{
Valid: true,
WeakValidation: true,
Algorithm: integrityAlgorithm,
Message: "ok (unkeyed local hash chain; weak validation)",
}
if err := s.ensureOpen(); err != nil {
return report, err
}
rows, err := s.db.QueryContext(context.Background(), selectEventColumns+` ORDER BY sequence ASC`)
if err != nil {
return report, fmt.Errorf("read sql audit chain: %w", err)
}
defer rows.Close()
previousHash := ""
var previousSequence int64
firstRecord := true
for rows.Next() {
event, err := scanEvent(rows)
if err != nil {
return report, err
}
report.CheckedRecords++
if firstRecord {
report.FirstSequence = event.Sequence
}
report.LastSequence = event.Sequence
if event.Sequence <= 0 {
return invalidIntegrityReport(report, event.Sequence, "sequence must be positive"), nil
}
isFirst := firstRecord
if isFirst {
report.PartialChain = event.Sequence > 1 || event.PrevHash != ""
report.TruncatedPrefix = report.PartialChain
if report.PartialChain {
report.Message = "ok (partial chain after prefix retention; unkeyed weak validation)"
}
} else if event.Sequence <= previousSequence {
return invalidIntegrityReport(report, event.Sequence, "sequence order is invalid"), nil
}
if !isFirst && event.PrevHash != previousHash {
return invalidIntegrityReport(report, event.Sequence, "previous hash does not match"), nil
}
expected, err := calculateEventHash(event)
if err != nil {
return report, err
}
if event.Hash != expected {
return invalidIntegrityReport(report, event.Sequence, "record hash does not match"), nil
}
previousSequence = event.Sequence
previousHash = event.Hash
firstRecord = false
}
if err := rows.Err(); err != nil {
return report, fmt.Errorf("iterate sql audit chain: %w", err)
}
return report, nil
}
func (s *Store) ensureOpen() error {
if s == nil || s.db == nil {
return ErrClosed
}
return nil
}
func (s *Store) withImmediate(operation func(*sql.Conn) error) (resultErr error) {
if err := s.ensureOpen(); err != nil {
return err
}
ctx := context.Background()
conn, err := s.db.Conn(ctx)
if err != nil {
return fmt.Errorf("acquire sql audit connection: %w", err)
}
defer func() { resultErr = errors.Join(resultErr, conn.Close()) }()
if _, err := conn.ExecContext(ctx, "BEGIN IMMEDIATE"); err != nil {
return fmt.Errorf("begin sql audit transaction: %w", err)
}
committed := false
defer func() {
if !committed {
_, rollbackErr := conn.ExecContext(context.Background(), "ROLLBACK")
resultErr = errors.Join(resultErr, rollbackErr)
}
}()
if operation != nil {
if err := operation(conn); err != nil {
return err
}
}
if _, err := conn.ExecContext(ctx, "COMMIT"); err != nil {
return fmt.Errorf("commit sql audit transaction: %w", err)
}
committed = true
return nil
}
type queryRower interface {
QueryRowContext(context.Context, string, ...any) *sql.Row
}
func getSettingsFrom(queryer queryRower) (Settings, error) {
var enabled int
var settings Settings
err := queryer.QueryRowContext(context.Background(), `SELECT enabled, capture_mode, retention_days, max_records
FROM sql_audit_settings WHERE singleton=1`).Scan(
&enabled, &settings.CaptureMode, &settings.RetentionDays, &settings.MaxRecords,
)
if err != nil {
return Settings{}, fmt.Errorf("read sql audit settings: %w", err)
}
settings.Enabled = enabled != 0
normalized, err := normalizeSettings(settings)
if err != nil {
return Settings{}, fmt.Errorf("invalid persisted sql audit settings: %w", err)
}
return normalized, nil
}
func normalizeSettings(settings Settings) (Settings, error) {
settings.CaptureMode = strings.ToLower(strings.TrimSpace(settings.CaptureMode))
if settings.CaptureMode == "" {
settings.CaptureMode = CaptureModeRedacted
}
if settings.CaptureMode != CaptureModeRedacted && settings.CaptureMode != CaptureModeMetadata {
return Settings{}, fmt.Errorf("unsupported SQL audit capture mode %q", settings.CaptureMode)
}
if settings.RetentionDays <= 0 {
settings.RetentionDays = defaultRetentionDays
}
if settings.RetentionDays > maxRetentionDays {
return Settings{}, fmt.Errorf("SQL audit retention days cannot exceed %d", maxRetentionDays)
}
if settings.MaxRecords <= 0 {
settings.MaxRecords = defaultMaxRecords
}
if settings.MaxRecords > maxConfiguredEvents {
return Settings{}, fmt.Errorf("SQL audit max records cannot exceed %d", maxConfiguredEvents)
}
return settings, nil
}
func normalizeFilter(filter Filter) Filter {
filter.Search = truncateRunes(strings.TrimSpace(filter.Search), 512)
filter.ConnectionID = strings.TrimSpace(filter.ConnectionID)
filter.Database = strings.TrimSpace(filter.Database)
filter.DBType = strings.ToLower(strings.TrimSpace(filter.DBType))
filter.EventType = strings.ToLower(strings.TrimSpace(filter.EventType))
filter.Status = strings.ToLower(strings.TrimSpace(filter.Status))
filter.TransactionID = strings.TrimSpace(filter.TransactionID)
filter.Source = strings.ToLower(strings.TrimSpace(filter.Source))
if filter.Page <= 0 {
filter.Page = 1
}
if filter.PageSize <= 0 {
filter.PageSize = defaultPageSize
}
if filter.PageSize > maxPageSize {
filter.PageSize = maxPageSize
}
return filter
}
func buildFilterWhere(filter Filter) (string, []any) {
conditions := make([]string, 0, 10)
args := make([]any, 0, 12)
if filter.Search != "" {
pattern := "%" + escapeLike(filter.Search) + "%"
conditions = append(conditions, `(sql_text LIKE ? ESCAPE '\' OR error_text LIKE ? ESCAPE '\'
OR connection_id LIKE ? ESCAPE '\' OR connection_fingerprint LIKE ? ESCAPE '\'
OR database_name LIKE ? ESCAPE '\' OR db_type LIKE ? ESCAPE '\'
OR query_id LIKE ? ESCAPE '\' OR transaction_id LIKE ? ESCAPE '\'
OR sql_fingerprint LIKE ? ESCAPE '\' OR source LIKE ? ESCAPE '\')`)
for range 10 {
args = append(args, pattern)
}
}
appendExact := func(column string, value string) {
if value == "" {
return
}
conditions = append(conditions, column+" = ?")
args = append(args, value)
}
appendExact("connection_id", filter.ConnectionID)
appendExact("database_name", filter.Database)
appendExact("db_type", filter.DBType)
appendExact("event_type", filter.EventType)
appendExact("status", filter.Status)
appendExact("transaction_id", filter.TransactionID)
appendExact("source", filter.Source)
if filter.FromTimestamp > 0 {
conditions = append(conditions, "timestamp >= ?")
args = append(args, filter.FromTimestamp)
}
if filter.ToTimestamp > 0 {
conditions = append(conditions, "timestamp <= ?")
args = append(args, filter.ToTimestamp)
}
if len(conditions) == 0 {
return "", args
}
return " WHERE " + strings.Join(conditions, " AND "), args
}
func escapeLike(value string) string {
value = strings.ReplaceAll(value, `\`, `\\`)
value = strings.ReplaceAll(value, `%`, `\%`)
return strings.ReplaceAll(value, `_`, `\_`)
}
func pruneLocked(conn *sql.Conn, settings Settings, now time.Time, reserve int64) (int64, error) {
var deleted int64
cutoff := now.Add(-time.Duration(settings.RetentionDays) * 24 * time.Hour).UnixMilli()
expired, err := deleteTimestampPrefixLocked(conn, cutoff)
if err != nil {
return 0, fmt.Errorf("prune expired sql audit events: %w", err)
}
deleted += expired
var current int64
if err := conn.QueryRowContext(context.Background(), `SELECT COUNT(*) FROM sql_audit_events`).Scan(&current); err != nil {
return 0, fmt.Errorf("count sql audit events: %w", err)
}
target := int64(settings.MaxRecords) - reserve
if target < 0 {
target = 0
}
if current > target {
removeCount := current - target
result, err := conn.ExecContext(context.Background(), `DELETE FROM sql_audit_events WHERE sequence IN (
SELECT sequence FROM sql_audit_events ORDER BY sequence ASC LIMIT ?
)`, removeCount)
if err != nil {
return 0, fmt.Errorf("enforce sql audit record limit: %w", err)
}
if count, err := result.RowsAffected(); err == nil {
deleted += count
}
}
return deleted, nil
}
func deleteAllLocked(conn *sql.Conn) (int64, error) {
result, err := conn.ExecContext(context.Background(), `DELETE FROM sql_audit_events`)
if err != nil {
return 0, fmt.Errorf("clear SQL audit events: %w", err)
}
deleted, err := result.RowsAffected()
if err != nil {
return 0, fmt.Errorf("count cleared SQL audit events: %w", err)
}
return deleted, nil
}
// deleteTimestampPrefixLocked never removes an event after the first retained
// sequence, even when a caller supplied an out-of-order timestamp. This keeps
// every remaining adjacency and record hash untouched.
func deleteTimestampPrefixLocked(conn *sql.Conn, beforeTimestamp int64) (int64, error) {
var firstRetainedSequence int64
err := conn.QueryRowContext(context.Background(), `SELECT sequence FROM sql_audit_events
WHERE timestamp >= ? ORDER BY sequence ASC LIMIT 1`, beforeTimestamp).Scan(&firstRetainedSequence)
if errors.Is(err, sql.ErrNoRows) {
return deleteAllLocked(conn)
}
if err != nil {
return 0, fmt.Errorf("locate SQL audit retention boundary: %w", err)
}
result, err := conn.ExecContext(context.Background(),
`DELETE FROM sql_audit_events WHERE sequence < ?`, firstRetainedSequence)
if err != nil {
return 0, fmt.Errorf("delete SQL audit sequence prefix: %w", err)
}
deleted, err := result.RowsAffected()
if err != nil {
return 0, fmt.Errorf("count deleted SQL audit sequence prefix: %w", err)
}
return deleted, nil
}
type canonicalEvent struct {
Version string `json:"version"`
ID string `json:"id"`
Sequence int64 `json:"sequence"`
Timestamp int64 `json:"timestamp"`
EventType string `json:"eventType"`
Status string `json:"status"`
ConnectionID string `json:"connectionId"`
ConnectionFingerprint string `json:"connectionFingerprint"`
DBType string `json:"dbType"`
Database string `json:"database"`
QueryID string `json:"queryId"`
TransactionID string `json:"transactionId"`
Source string `json:"source"`
BoundaryMode string `json:"boundaryMode"`
CommitMode string `json:"commitMode"`
SQLText string `json:"sqlText"`
SQLRedacted bool `json:"sqlRedacted"`
SQLFingerprint string `json:"sqlFingerprint"`
StatementIndex int `json:"statementIndex"`
StatementCount int `json:"statementCount"`
DurationMs int64 `json:"durationMs"`
RowsAffected int64 `json:"rowsAffected"`
RowsReturned int64 `json:"rowsReturned"`
Error string `json:"error"`
PrevHash string `json:"prevHash"`
}
func calculateEventHash(event Event) (string, error) {
payload, err := json.Marshal(canonicalEvent{
Version: chainVersion,
ID: event.ID,
Sequence: event.Sequence,
Timestamp: event.Timestamp,
EventType: event.EventType,
Status: event.Status,
ConnectionID: event.ConnectionID,
ConnectionFingerprint: event.ConnectionFingerprint,
DBType: event.DBType,
Database: event.Database,
QueryID: event.QueryID,
TransactionID: event.TransactionID,
Source: event.Source,
BoundaryMode: event.BoundaryMode,
CommitMode: event.CommitMode,
SQLText: event.SQLText,
SQLRedacted: event.SQLRedacted,
SQLFingerprint: event.SQLFingerprint,
StatementIndex: event.StatementIndex,
StatementCount: event.StatementCount,
DurationMs: event.DurationMs,
RowsAffected: event.RowsAffected,
RowsReturned: event.RowsReturned,
Error: event.Error,
PrevHash: event.PrevHash,
})
if err != nil {
return "", fmt.Errorf("encode canonical SQL audit event: %w", err)
}
digest := sha256.Sum256(payload)
return hex.EncodeToString(digest[:]), nil
}
func invalidIntegrityReport(report IntegrityReport, sequence int64, message string) IntegrityReport {
report.Valid = false
report.InvalidSequence = sequence
report.Message = message + " (unkeyed local hash chain; weak validation)"
return report
}
const selectEventColumns = `SELECT sequence, id, timestamp, event_type, status,
connection_id, connection_fingerprint, db_type, database_name, query_id,
transaction_id, source, boundary_mode, commit_mode, sql_text, sql_redacted,
sql_fingerprint, statement_index, statement_count, duration_ms, rows_affected,
rows_returned, error_text, prev_hash, record_hash FROM sql_audit_events`
type rowScanner interface {
Scan(...any) error
}
func scanEvent(scanner rowScanner) (Event, error) {
var event Event
var sqlRedacted int
if err := scanner.Scan(
&event.Sequence, &event.ID, &event.Timestamp, &event.EventType, &event.Status,
&event.ConnectionID, &event.ConnectionFingerprint, &event.DBType, &event.Database,
&event.QueryID, &event.TransactionID, &event.Source, &event.BoundaryMode,
&event.CommitMode, &event.SQLText, &sqlRedacted, &event.SQLFingerprint,
&event.StatementIndex, &event.StatementCount, &event.DurationMs,
&event.RowsAffected, &event.RowsReturned, &event.Error, &event.PrevHash, &event.Hash,
); err != nil {
return Event{}, fmt.Errorf("scan SQL audit event: %w", err)
}
event.SQLRedacted = sqlRedacted != 0
return event, nil
}
func boolToInt(value bool) int {
if value {
return 1
}
return 0
}

View File

@@ -0,0 +1,652 @@
package sqlaudit
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"sync"
"testing"
"time"
)
func openTestStore(t *testing.T) *Store {
t.Helper()
store, err := Open(filepath.Join(t.TempDir(), "audit", "sql_audit.db"))
if err != nil {
t.Fatalf("Open returned error: %v", err)
}
t.Cleanup(func() {
if err := store.Close(); err != nil && !errors.Is(err, ErrClosed) {
t.Errorf("Close returned error: %v", err)
}
})
return store
}
func sampleEvent(id string, timestamp int64) Event {
return Event{
ID: id,
Timestamp: timestamp,
EventType: "query",
Status: "success",
ConnectionID: "conn-main",
ConnectionFingerprint: "postgres://admin:raw-secret@db.example/app",
DBType: "mysql",
Database: "analytics",
QueryID: "query-1",
Source: "query_editor",
BoundaryMode: BoundaryModeDriverAPI,
SQLText: "SELECT * FROM users WHERE id = 42 AND token = 'raw-token'",
DurationMs: 25,
RowsReturned: 1,
}
}
func TestOpenConfiguresSQLiteAndDefaultSettings(t *testing.T) {
store := openTestStore(t)
settings, err := store.GetSettings()
if err != nil {
t.Fatalf("GetSettings returned error: %v", err)
}
if settings != DefaultSettings() {
t.Fatalf("unexpected defaults: %#v", settings)
}
var journalMode string
if err := store.db.QueryRow(`PRAGMA journal_mode`).Scan(&journalMode); err != nil {
t.Fatalf("read journal mode: %v", err)
}
if strings.ToLower(journalMode) != "wal" {
t.Fatalf("journal mode must be WAL, got %q", journalMode)
}
var synchronous int
if err := store.db.QueryRow(`PRAGMA synchronous`).Scan(&synchronous); err != nil {
t.Fatalf("read synchronous mode: %v", err)
}
if synchronous != 2 {
t.Fatalf("synchronous must be FULL (2), got %d", synchronous)
}
if got := store.db.Stats().MaxOpenConnections; got != 4 {
t.Fatalf("MaxOpenConnections=%d, want 4", got)
}
connections := make([]interface{ Close() error }, 0, 4)
for index := 0; index < 4; index++ {
conn, err := store.db.Conn(context.Background())
if err != nil {
t.Fatalf("acquire pooled connection %d: %v", index+1, err)
}
connections = append(connections, conn)
var busyTimeout, foreignKeys, connectionSynchronous int
var connectionJournal string
if err := conn.QueryRowContext(context.Background(), `PRAGMA busy_timeout`).Scan(&busyTimeout); err != nil {
t.Fatalf("read connection %d busy_timeout: %v", index+1, err)
}
if err := conn.QueryRowContext(context.Background(), `PRAGMA foreign_keys`).Scan(&foreignKeys); err != nil {
t.Fatalf("read connection %d foreign_keys: %v", index+1, err)
}
if err := conn.QueryRowContext(context.Background(), `PRAGMA synchronous`).Scan(&connectionSynchronous); err != nil {
t.Fatalf("read connection %d synchronous: %v", index+1, err)
}
if err := conn.QueryRowContext(context.Background(), `PRAGMA journal_mode`).Scan(&connectionJournal); err != nil {
t.Fatalf("read connection %d journal_mode: %v", index+1, err)
}
if busyTimeout != 5000 || foreignKeys != 1 || connectionSynchronous != 2 || strings.ToLower(connectionJournal) != "wal" {
t.Fatalf("connection %d PRAGMAs unexpected: busy=%d foreign=%d sync=%d journal=%q",
index+1, busyTimeout, foreignKeys, connectionSynchronous, connectionJournal)
}
}
for _, conn := range connections {
if err := conn.Close(); err != nil {
t.Fatalf("close pooled connection: %v", err)
}
}
if runtime.GOOS != "windows" {
dirInfo, err := os.Stat(filepath.Dir(store.Path()))
if err != nil {
t.Fatalf("stat audit directory: %v", err)
}
if got := dirInfo.Mode().Perm(); got != 0o700 {
t.Fatalf("audit directory mode = %#o, want 0700", got)
}
fileInfo, err := os.Stat(store.Path())
if err != nil {
t.Fatalf("stat audit database: %v", err)
}
if got := fileInfo.Mode().Perm(); got != 0o600 {
t.Fatalf("audit database mode = %#o, want 0600", got)
}
}
}
func TestOpenSupportsURIReservedCharactersInPath(t *testing.T) {
path := filepath.Join(t.TempDir(), "audit space # percent% amp&", "sql audit.db")
store, err := Open(path)
if err != nil {
t.Fatalf("Open reserved-character path returned error: %v", err)
}
if err := store.Append(sampleEvent("reserved-path", time.Now().UnixMilli())); err != nil {
_ = store.Close()
t.Fatalf("Append reserved-character path returned error: %v", err)
}
if err := store.Close(); err != nil {
t.Fatalf("Close reserved-character path returned error: %v", err)
}
if _, err := os.Stat(path); err != nil {
t.Fatalf("SQLite database was not created at literal path %q: %v", path, err)
}
if info, err := os.Stat(path + "-wal"); err == nil && info.Size() > 0 {
t.Fatalf("Close should checkpoint/truncate WAL before migration, size=%d", info.Size())
} else if err != nil && !os.IsNotExist(err) {
t.Fatalf("stat WAL after Close: %v", err)
}
}
func TestAppendSanitizesAndQueryFiltersWithSummary(t *testing.T) {
store := openTestStore(t)
now := time.Now().UnixMilli()
first := sampleEvent("event-1", now-100)
first.TransactionID = "tx-1"
if err := store.Append(first); err != nil {
t.Fatalf("Append first returned error: %v", err)
}
second := sampleEvent("event-2", now)
second.Status = "error"
second.TransactionID = "tx-2"
second.Error = "password=driver-secret failed near 'raw-token'"
if err := store.Append(second); err != nil {
t.Fatalf("Append second returned error: %v", err)
}
page, err := store.Query(Filter{Database: "analytics", Page: 1, PageSize: 1})
if err != nil {
t.Fatalf("Query returned error: %v", err)
}
if page.Total != 2 || len(page.Items) != 1 || page.Items[0].ID != "event-2" {
t.Fatalf("unexpected page: %#v", page)
}
if page.Summary.TotalEvents != 2 || page.Summary.SuccessCount != 1 ||
page.Summary.ErrorCount != 1 || page.Summary.TransactionCount != 2 {
t.Fatalf("unexpected summary: %#v", page.Summary)
}
event := page.Items[0]
for _, secret := range []string{"42", "raw-token", "driver-secret", "admin", "raw-secret", "db.example"} {
if strings.Contains(event.SQLText+event.Error+event.ConnectionFingerprint, secret) {
t.Fatalf("stored audit event leaked %q: %#v", secret, event)
}
}
if !event.SQLRedacted || event.SQLFingerprint == "" || len(event.ConnectionFingerprint) != 64 {
t.Fatalf("event was not safely normalized: %#v", event)
}
errorPage, err := store.Query(Filter{Status: "error", Search: "password", PageSize: 10})
if err != nil || errorPage.Total != 1 || len(errorPage.Items) != 1 {
t.Fatalf("filtered query failed: page=%#v err=%v", errorPage, err)
}
assertAuditStorageFilesDoNotContain(t, store.Path(), []string{"raw-token", "driver-secret", "raw-secret"})
}
func TestQuerySupportsContractFiltersAndEscapesSearchWildcards(t *testing.T) {
store := openTestStore(t)
now := time.Now().UnixMilli()
target := sampleEvent("filter-target", now)
target.EventType = "TRANSACTION_COMMIT"
target.Status = "SUCCESS"
target.TransactionID = "tx-filter"
target.Source = "SYSTEM"
target.DBType = "MYSQL"
if err := store.Append(target); err != nil {
t.Fatalf("Append target returned error: %v", err)
}
other := sampleEvent("filter-other", now+100)
other.ConnectionID = "conn-other"
other.Database = "reporting"
other.TransactionID = "tx-other"
if err := store.Append(other); err != nil {
t.Fatalf("Append other returned error: %v", err)
}
page, err := store.Query(Filter{
ConnectionID: "conn-main",
Database: "analytics",
DBType: "MYSQL",
EventType: "TRANSACTION_COMMIT",
Status: "SUCCESS",
TransactionID: "tx-filter",
Source: "SYSTEM",
FromTimestamp: now - 1,
ToTimestamp: now + 1,
PageSize: 10,
})
if err != nil || page.Total != 1 || len(page.Items) != 1 || page.Items[0].ID != "filter-target" {
t.Fatalf("contract filters did not isolate target: page=%#v err=%v", page, err)
}
page, err = store.Query(Filter{Search: `%_' OR 1=1 --`, PageSize: 10})
if err != nil || page.Total != 0 {
t.Fatalf("literal wildcard/injection-like search should not broaden results: page=%#v err=%v", page, err)
}
all, err := store.Query(Filter{PageSize: 10})
if err != nil || len(all.Items) == 0 {
t.Fatalf("load events for fingerprint search: page=%#v err=%v", all, err)
}
fingerprintPrefix := all.Items[0].SQLFingerprint[:12]
page, err = store.Query(Filter{Search: fingerprintPrefix, PageSize: 10})
if err != nil || page.Total == 0 {
t.Fatalf("fingerprint search should find matching events: page=%#v err=%v", page, err)
}
}
func TestMetadataAndDisabledCaptureModes(t *testing.T) {
store := openTestStore(t)
settings := DefaultSettings()
settings.CaptureMode = CaptureModeMetadata
if err := store.UpdateSettings(settings); err != nil {
t.Fatalf("UpdateSettings(metadata) returned error: %v", err)
}
if err := store.Append(sampleEvent("metadata-event", time.Now().UnixMilli())); err != nil {
t.Fatalf("Append metadata event returned error: %v", err)
}
page, err := store.Query(Filter{PageSize: 10})
if err != nil || len(page.Items) != 1 {
t.Fatalf("query metadata event: page=%#v err=%v", page, err)
}
if page.Items[0].SQLText != "" || page.Items[0].SQLFingerprint == "" || !page.Items[0].SQLRedacted {
t.Fatalf("metadata capture persisted SQL text: %#v", page.Items[0])
}
settings.Enabled = false
if err := store.UpdateSettings(settings); err != nil {
t.Fatalf("UpdateSettings(disabled) returned error: %v", err)
}
if err := store.Append(sampleEvent("disabled-event", time.Now().UnixMilli())); err != nil {
t.Fatalf("disabled Append should be a no-op: %v", err)
}
page, err = store.Query(Filter{PageSize: 10})
if err != nil || page.Total != 1 {
t.Fatalf("disabled capture should not append: page=%#v err=%v", page, err)
}
}
func TestControlBoundariesAreAtomicAndBypassDisabledCapture(t *testing.T) {
store := openTestStore(t)
settings := DefaultSettings()
settings.Enabled = false
settings.CaptureMode = CaptureModeMetadata
control := Event{ID: "settings-control", EventType: "audit_settings_change", Status: "success", Source: "audit_control"}
if err := store.UpdateSettingsWithControl(settings, control); err != nil {
t.Fatalf("UpdateSettingsWithControl returned error: %v", err)
}
if err := store.Append(sampleEvent("disabled-ordinary", time.Now().UnixMilli())); err != nil {
t.Fatalf("disabled ordinary append returned error: %v", err)
}
page, err := store.Query(Filter{PageSize: 10})
if err != nil || page.Total != 1 || page.Items[0].ID != "settings-control" {
t.Fatalf("disabled control boundary was not retained: page=%#v err=%v", page, err)
}
if !strings.Contains(page.Items[0].SQLText, "FROM_ENABLED_ON TO_ENABLED_OFF") {
t.Fatalf("settings control boundary lacks safe old/new metadata: %q", page.Items[0].SQLText)
}
deleted, err := store.ClearWithControl(0, Event{ID: "clear-control", EventType: "audit_clear", Status: "success", Source: "audit_control"})
if err != nil || deleted != 1 {
t.Fatalf("ClearWithControl deleted=%d err=%v, want 1", deleted, err)
}
page, err = store.Query(Filter{PageSize: 10})
if err != nil || page.Total != 1 || page.Items[0].ID != "clear-control" || page.Items[0].RowsAffected != 1 {
t.Fatalf("clear control boundary was not atomically retained: page=%#v err=%v", page, err)
}
if page.Items[0].SQLText != "AUDIT CLEAR ALL" {
t.Fatalf("metadata capture erased the clear control descriptor: %q", page.Items[0].SQLText)
}
}
func TestAppendNeverPersistsRedisOrMessagePayloadSecrets(t *testing.T) {
store := openTestStore(t)
now := time.Now().UnixMilli()
redisEvent := sampleEvent("redis-secret", now)
redisEvent.DBType = "redis"
redisEvent.SQLText = `HSET account:42 email private@example.test token redis-token-secret`
redisEvent.Error = `AUTH failed password=redis-error-secret`
if err := store.Append(redisEvent); err != nil {
t.Fatalf("Append Redis event returned error: %v", err)
}
mqEvent := sampleEvent("mq-secret", now+1)
mqEvent.DBType = "rocketmq"
mqEvent.SQLText = `PUBLISH orders {"token":"mq-payload-secret","card":"4111111111111111"}`
mqEvent.Error = `publish rejected token=mq-error-secret`
if err := store.Append(mqEvent); err != nil {
t.Fatalf("Append MQ event returned error: %v", err)
}
mqEventSecond := sampleEvent("mq-secret-second", now+2)
mqEventSecond.DBType = "rocketmq"
mqEventSecond.SQLText = `PUBLISH orders {"token":"different-low-entropy-secret"}`
if err := store.Append(mqEventSecond); err != nil {
t.Fatalf("Append second MQ event returned error: %v", err)
}
page, err := store.Query(Filter{PageSize: 10})
if err != nil || len(page.Items) != 3 {
t.Fatalf("query non-SQL audit events: page=%#v err=%v", page, err)
}
byID := map[string]Event{}
for _, event := range page.Items {
byID[event.ID] = event
}
if got := byID["redis-secret"].SQLText; got != "HSET account:42 email ? token ?" {
t.Fatalf("unexpected Redis audit text: %q", got)
}
if got := byID["mq-secret"].SQLText; got != "" {
t.Fatalf("message payload must be metadata-only, got %q", got)
}
if byID["mq-secret"].SQLFingerprint == "" {
t.Fatal("metadata-only message event still needs a safe fingerprint")
}
if byID["mq-secret"].SQLFingerprint != byID["mq-secret-second"].SQLFingerprint {
t.Fatalf("metadata-only fingerprint must not depend on message payload: first=%s second=%s",
byID["mq-secret"].SQLFingerprint, byID["mq-secret-second"].SQLFingerprint)
}
assertAuditStorageFilesDoNotContain(t, store.Path(), []string{
"private@example.test", "redis-token-secret", "redis-error-secret",
"mq-payload-secret", "4111111111111111", "mq-error-secret", "different-low-entropy-secret",
})
}
func TestIntegrityDetectsTampering(t *testing.T) {
store := openTestStore(t)
now := time.Now().UnixMilli()
for index := 0; index < 2; index++ {
if err := store.Append(sampleEvent("integrity-"+string(rune('a'+index)), now+int64(index))); err != nil {
t.Fatalf("Append returned error: %v", err)
}
}
report, err := store.VerifyIntegrity()
if err != nil || !report.Valid || !report.WeakValidation || report.CheckedRecords != 2 {
t.Fatalf("unexpected valid integrity report: %#v err=%v", report, err)
}
if _, err := store.db.Exec(`UPDATE sql_audit_events SET sql_text='tampered' WHERE sequence=1`); err != nil {
t.Fatalf("tamper test database: %v", err)
}
report, err = store.VerifyIntegrity()
if err != nil || report.Valid || report.InvalidSequence != 1 {
t.Fatalf("tampering was not detected: %#v err=%v", report, err)
}
}
func TestClearAndRecordLimitPreserveRemainingHashes(t *testing.T) {
store := openTestStore(t)
now := time.Now().UnixMilli()
for index := 0; index < 3; index++ {
if err := store.Append(sampleEvent("clear-"+string(rune('a'+index)), now-3000+int64(index*1000))); err != nil {
t.Fatalf("Append returned error: %v", err)
}
}
beforeClear, err := store.Query(Filter{PageSize: 10})
if err != nil || len(beforeClear.Items) != 3 {
t.Fatalf("query before clear: page=%#v err=%v", beforeClear, err)
}
retainedHash := beforeClear.Items[0].Hash
deleted, err := store.Clear(now - 1500)
if err != nil || deleted != 2 {
t.Fatalf("Clear deleted=%d err=%v, want 2", deleted, err)
}
page, err := store.Query(Filter{PageSize: 10})
if err != nil || len(page.Items) != 1 || page.Items[0].PrevHash == "" {
t.Fatalf("clear rewrote or lost the retained chain anchor: page=%#v err=%v", page, err)
}
if page.Items[0].Hash != retainedHash {
t.Fatalf("clear must not rewrite retained event hash: before=%s after=%s", retainedHash, page.Items[0].Hash)
}
if report, err := store.VerifyIntegrity(); err != nil || !report.Valid || !report.PartialChain || !report.TruncatedPrefix {
t.Fatalf("chain invalid after clear: report=%#v err=%v", report, err)
}
settings := DefaultSettings()
settings.MaxRecords = 2
if err := store.UpdateSettings(settings); err != nil {
t.Fatalf("UpdateSettings returned error: %v", err)
}
for index := 0; index < 3; index++ {
if err := store.Append(sampleEvent("limited-"+string(rune('a'+index)), now+int64(index))); err != nil {
t.Fatalf("Append limited event returned error: %v", err)
}
}
page, err = store.Query(Filter{PageSize: 10})
if err != nil || page.Total != 2 {
t.Fatalf("record limit not enforced: page=%#v err=%v", page, err)
}
if report, err := store.VerifyIntegrity(); err != nil || !report.Valid || !report.PartialChain {
t.Fatalf("chain invalid after record-limit pruning: report=%#v err=%v", report, err)
}
}
func TestClearDeletesOnlyContiguousPrefixForOutOfOrderTimestamps(t *testing.T) {
store := openTestStore(t)
now := time.Now().UnixMilli()
for _, event := range []Event{
sampleEvent("prefix-old", now-10_000),
sampleEvent("prefix-new", now),
sampleEvent("later-sequence-old-time", now-20_000),
} {
if err := store.Append(event); err != nil {
t.Fatalf("Append returned error: %v", err)
}
}
before, err := store.Query(Filter{PageSize: 10})
if err != nil || len(before.Items) != 3 {
t.Fatalf("query before clear: page=%#v err=%v", before, err)
}
hashes := map[string]string{}
for _, event := range before.Items {
hashes[event.ID] = event.Hash
}
deleted, err := store.Clear(now - 5_000)
if err != nil || deleted != 1 {
t.Fatalf("Clear should delete only one contiguous prefix event: deleted=%d err=%v", deleted, err)
}
after, err := store.Query(Filter{PageSize: 10})
if err != nil || len(after.Items) != 2 {
t.Fatalf("query after clear: page=%#v err=%v", after, err)
}
for _, event := range after.Items {
if event.Hash != hashes[event.ID] {
t.Fatalf("retained event %s hash was rewritten", event.ID)
}
}
if report, err := store.VerifyIntegrity(); err != nil || !report.Valid || !report.PartialChain {
t.Fatalf("partial chain should remain verifiable: report=%#v err=%v", report, err)
}
}
func TestConcurrentAppendKeepsUniqueOrderedChain(t *testing.T) {
store := openTestStore(t)
const count = 32
now := time.Now().UnixMilli()
var wait sync.WaitGroup
errorsCh := make(chan error, count)
for index := 0; index < count; index++ {
wait.Add(1)
go func(index int) {
defer wait.Done()
event := sampleEvent("concurrent-"+time.UnixMilli(int64(index)).Format("150405.000000000"), now+int64(index))
errorsCh <- store.Append(event)
}(index)
}
wait.Wait()
close(errorsCh)
for err := range errorsCh {
if err != nil {
t.Fatalf("concurrent Append returned error: %v", err)
}
}
page, err := store.Query(Filter{PageSize: count})
if err != nil || page.Total != count {
t.Fatalf("concurrent events missing: total=%d err=%v", page.Total, err)
}
seen := make(map[int64]struct{}, count)
for _, event := range page.Items {
if _, duplicate := seen[event.Sequence]; duplicate {
t.Fatalf("duplicate sequence %d", event.Sequence)
}
seen[event.Sequence] = struct{}{}
}
if report, err := store.VerifyIntegrity(); err != nil || !report.Valid || report.CheckedRecords != count {
t.Fatalf("concurrent chain invalid: report=%#v err=%v", report, err)
}
}
func TestWALReaderDoesNotBlockSynchronousAppend(t *testing.T) {
store := openTestStore(t)
now := time.Now().UnixMilli()
if err := store.Append(sampleEvent("wal-seed", now)); err != nil {
t.Fatalf("Append seed returned error: %v", err)
}
reader, err := store.db.Conn(context.Background())
if err != nil {
t.Fatalf("acquire reader connection: %v", err)
}
defer reader.Close()
if _, err := reader.ExecContext(context.Background(), `BEGIN`); err != nil {
t.Fatalf("begin reader transaction: %v", err)
}
rows, err := reader.QueryContext(context.Background(), `SELECT id FROM sql_audit_events ORDER BY sequence`)
if err != nil {
_, _ = reader.ExecContext(context.Background(), `ROLLBACK`)
t.Fatalf("open reader cursor: %v", err)
}
defer rows.Close()
if !rows.Next() {
t.Fatal("reader cursor should expose seed event")
}
var seedID string
if err := rows.Scan(&seedID); err != nil || seedID != "wal-seed" {
t.Fatalf("scan reader cursor id=%q err=%v", seedID, err)
}
done := make(chan error, 1)
go func() {
done <- store.Append(sampleEvent("wal-writer", now+1))
}()
select {
case err := <-done:
if err != nil {
t.Fatalf("Append while WAL reader active returned error: %v", err)
}
case <-time.After(2 * time.Second):
t.Fatal("active WAL reader blocked synchronous audit append")
}
if err := rows.Close(); err != nil {
t.Fatalf("close reader rows: %v", err)
}
if _, err := reader.ExecContext(context.Background(), `ROLLBACK`); err != nil {
t.Fatalf("rollback reader transaction: %v", err)
}
}
func TestAppendBatchIsAtomicAndKeepsOneContinuousChain(t *testing.T) {
store := openTestStore(t)
now := time.Now().UnixMilli()
if err := store.Append(sampleEvent("before-batch", now)); err != nil {
t.Fatalf("Append before batch returned error: %v", err)
}
batch := []Event{
sampleEvent("batch-1", now+1),
sampleEvent("batch-2", now+2),
sampleEvent("batch-3", now+3),
}
if err := store.AppendBatch(batch); err != nil {
t.Fatalf("AppendBatch returned error: %v", err)
}
page, err := store.Query(Filter{PageSize: 10})
if err != nil || page.Total != 4 {
t.Fatalf("query after batch: page=%#v err=%v", page, err)
}
report, err := store.VerifyIntegrity()
if err != nil || !report.Valid || report.CheckedRecords != 4 {
t.Fatalf("integrity after batch: report=%#v err=%v", report, err)
}
duplicateBatch := []Event{
sampleEvent("atomic-new", now+4),
sampleEvent("batch-2", now+5),
}
if err := store.AppendBatch(duplicateBatch); err == nil {
t.Fatal("AppendBatch with duplicate ID should fail")
}
page, err = store.Query(Filter{Search: "atomic-new", PageSize: 10})
if err != nil || page.Total != 0 {
t.Fatalf("failed batch should roll back every event: page=%#v err=%v", page, err)
}
}
func TestAppendBatchLargerThanMaxRecordsFailsAtomically(t *testing.T) {
store := openTestStore(t)
settings := DefaultSettings()
settings.MaxRecords = 3
if err := store.UpdateSettings(settings); err != nil {
t.Fatalf("UpdateSettings returned error: %v", err)
}
now := time.Now().UnixMilli()
if err := store.Append(sampleEvent("existing-before-large-batch", now)); err != nil {
t.Fatalf("Append existing event returned error: %v", err)
}
batch := make([]Event, 0, 6)
for index := 0; index < 6; index++ {
batch = append(batch, sampleEvent(fmt.Sprintf("large-batch-%d", index), now+int64(index+1)))
}
if err := store.AppendBatch(batch); !errors.Is(err, ErrBatchExceedsMaxRecords) {
t.Fatalf("AppendBatch error = %v, want ErrBatchExceedsMaxRecords", err)
}
page, err := store.Query(Filter{PageSize: 10})
if err != nil {
t.Fatalf("Query returned error: %v", err)
}
if page.Total != 1 || len(page.Items) != 1 || page.Items[0].ID != "existing-before-large-batch" {
t.Fatalf("oversized batch must not modify existing audit records: %#v", page)
}
if report, err := store.VerifyIntegrity(); err != nil || !report.Valid || report.CheckedRecords != 1 {
t.Fatalf("large-batch chain invalid: report=%#v err=%v", report, err)
}
}
func TestSettingsValidationAndClosedStore(t *testing.T) {
store := openTestStore(t)
invalid := DefaultSettings()
invalid.CaptureMode = "full"
if err := store.UpdateSettings(invalid); err == nil {
t.Fatal("full/raw SQL capture mode must be rejected")
}
if err := store.Close(); err != nil {
t.Fatalf("Close returned error: %v", err)
}
if err := store.Append(sampleEvent("closed", time.Now().UnixMilli())); !errors.Is(err, ErrClosed) {
t.Fatalf("Append after Close error=%v, want ErrClosed", err)
}
}
func assertAuditStorageFilesDoNotContain(t *testing.T, databasePath string, secrets []string) {
t.Helper()
for _, path := range []string{databasePath, databasePath + "-wal", databasePath + "-shm"} {
payload, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
continue
}
t.Fatalf("read SQLite audit storage file %q: %v", path, err)
}
for _, secret := range secrets {
if strings.Contains(string(payload), secret) {
t.Fatalf("SQLite audit storage file %q contains raw secret %q", path, secret)
}
}
}
}

114
internal/sqlaudit/types.go Normal file
View File

@@ -0,0 +1,114 @@
package sqlaudit
const (
CaptureModeRedacted = "redacted"
CaptureModeMetadata = "metadata"
BoundaryModeDriverAPI = "driver_api"
BoundaryModeTextSQL = "text_sql"
BoundaryModeImplicit = "implicit"
BoundaryModeUnknown = "unknown"
CommitModeAuto = "auto"
CommitModeManual = "manual"
CommitModePending = "pending"
)
const (
defaultPageSize = 50
maxPageSize = 500
defaultRetentionDays = 30
defaultMaxRecords = 100_000
)
// Event is one immutable SQL audit fact. SQLText is always sanitized again by
// Store.Append; callers cannot use this type to persist raw SQL.
type Event struct {
ID string `json:"id"`
Sequence int64 `json:"sequence"`
Timestamp int64 `json:"timestamp"`
EventType string `json:"eventType"`
Status string `json:"status"`
ConnectionID string `json:"connectionId"`
ConnectionFingerprint string `json:"connectionFingerprint"`
DBType string `json:"dbType"`
Database string `json:"database"`
QueryID string `json:"queryId"`
TransactionID string `json:"transactionId"`
Source string `json:"source"`
BoundaryMode string `json:"boundaryMode"`
CommitMode string `json:"commitMode"`
SQLText string `json:"sqlText"`
SQLRedacted bool `json:"sqlRedacted"`
SQLFingerprint string `json:"sqlFingerprint"`
StatementIndex int `json:"statementIndex"`
StatementCount int `json:"statementCount"`
DurationMs int64 `json:"durationMs"`
RowsAffected int64 `json:"rowsAffected"`
RowsReturned int64 `json:"rowsReturned"`
Error string `json:"error"`
PrevHash string `json:"prevHash"`
Hash string `json:"hash"`
}
type Filter struct {
Search string `json:"search"`
ConnectionID string `json:"connectionId"`
Database string `json:"database"`
DBType string `json:"dbType"`
EventType string `json:"eventType"`
Status string `json:"status"`
TransactionID string `json:"transactionId"`
Source string `json:"source"`
FromTimestamp int64 `json:"fromTimestamp"`
ToTimestamp int64 `json:"toTimestamp"`
Page int `json:"page"`
PageSize int `json:"pageSize"`
}
type Summary struct {
TotalEvents int64 `json:"totalEvents"`
SuccessCount int64 `json:"successCount"`
ErrorCount int64 `json:"errorCount"`
TransactionCount int64 `json:"transactionCount"`
}
type Page struct {
Items []Event `json:"items"`
Total int64 `json:"total"`
Page int `json:"page"`
PageSize int `json:"pageSize"`
Summary Summary `json:"summary"`
}
type Settings struct {
Enabled bool `json:"enabled"`
CaptureMode string `json:"captureMode"`
RetentionDays int `json:"retentionDays"`
MaxRecords int `json:"maxRecords"`
}
func DefaultSettings() Settings {
return Settings{
Enabled: true,
CaptureMode: CaptureModeRedacted,
RetentionDays: defaultRetentionDays,
MaxRecords: defaultMaxRecords,
}
}
// IntegrityReport reports local SHA-256 chain consistency. WeakValidation is
// always true because an unkeyed chain detects accidental/local edits but does
// not stop an attacker who can rewrite both records and hashes.
type IntegrityReport struct {
Valid bool `json:"valid"`
WeakValidation bool `json:"weakValidation"`
PartialChain bool `json:"partialChain"`
TruncatedPrefix bool `json:"truncatedPrefix"`
Algorithm string `json:"algorithm"`
CheckedRecords int64 `json:"checkedRecords"`
FirstSequence int64 `json:"firstSequence,omitempty"`
LastSequence int64 `json:"lastSequence,omitempty"`
InvalidSequence int64 `json:"invalidSequence,omitempty"`
Message string `json:"message"`
}