mirror of
https://github.com/Syngnat/GoNavi.git
synced 2026-07-15 17:33:08 +08:00
✨ feat(sql-audit): 新增 SQL 审计中心并完善事务追踪
- 新增脱敏审计存储、筛选、保留策略、完整性校验与 JSON/CSV 导出 - 覆盖查询编辑器、事务、导入同步、对象操作、AI/MCP 与 Web 运行时入口 - 完善事务完整语句日志、Windows 快捷键映射及 SQL 分析布局 - 补充多语言、健康状态、数据目录迁移和回归测试
This commit is contained in:
235
internal/sqlaudit/export.go
Normal file
235
internal/sqlaudit/export.go
Normal file
@@ -0,0 +1,235 @@
|
||||
package sqlaudit
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var ErrUnsupportedExportFormat = errors.New("unsupported SQL audit export format")
|
||||
|
||||
var (
|
||||
ErrExportRecordLimit = errors.New("SQL audit export record limit exceeded")
|
||||
ErrExportSizeLimit = errors.New("SQL audit export size limit exceeded")
|
||||
)
|
||||
|
||||
const (
|
||||
maxExportRecords int64 = 100_000
|
||||
maxExportBytes = 64 * 1024 * 1024
|
||||
)
|
||||
|
||||
// BuildExport returns every event matching filter. Pagination fields are
|
||||
// intentionally ignored so exporting from a paged workbench does not silently
|
||||
// export only the visible page.
|
||||
func (s *Store) BuildExport(filter Filter, format string) ([]byte, error) {
|
||||
return s.buildExportWithLimits(filter, format, maxExportRecords, maxExportBytes)
|
||||
}
|
||||
|
||||
// BuildExportWithLimits applies caller-specific resource caps. It is used by
|
||||
// the browser server, whose RPC envelope creates additional copies of content.
|
||||
func (s *Store) BuildExportWithLimits(filter Filter, format string, recordLimit int64, byteLimit int) ([]byte, error) {
|
||||
if recordLimit <= 0 || recordLimit > maxExportRecords {
|
||||
recordLimit = maxExportRecords
|
||||
}
|
||||
if byteLimit <= 0 || byteLimit > maxExportBytes {
|
||||
byteLimit = maxExportBytes
|
||||
}
|
||||
return s.buildExportWithLimits(filter, format, recordLimit, byteLimit)
|
||||
}
|
||||
|
||||
func (s *Store) buildExportWithLimits(filter Filter, format string, recordLimit int64, byteLimit int) ([]byte, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
format = strings.ToLower(strings.TrimSpace(format))
|
||||
if format != "json" && format != "csv" {
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedExportFormat, format)
|
||||
}
|
||||
filter = normalizeFilter(filter)
|
||||
filter.Page = 0
|
||||
filter.PageSize = 0
|
||||
events, err := s.loadExportEvents(filter, recordLimit, byteLimit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
switch format {
|
||||
case "json":
|
||||
output := newBoundedBuffer(byteLimit)
|
||||
if _, err := output.Write([]byte{'['}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for index, event := range events {
|
||||
payload, marshalErr := json.Marshal(event)
|
||||
if marshalErr != nil {
|
||||
return nil, fmt.Errorf("encode SQL audit JSON export: %w", marshalErr)
|
||||
}
|
||||
if index > 0 {
|
||||
if _, err := output.Write([]byte{','}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if _, err := output.Write(payload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if _, err := output.Write([]byte("]\n")); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return output.Bytes(), nil
|
||||
|
||||
case "csv":
|
||||
output := newBoundedBuffer(byteLimit)
|
||||
writer := csv.NewWriter(output)
|
||||
if err := writer.Write(exportCSVHeader); err != nil {
|
||||
return nil, fmt.Errorf("write SQL audit CSV header: %w", err)
|
||||
}
|
||||
for _, event := range events {
|
||||
if err := writer.Write(eventCSVRecord(event)); err != nil {
|
||||
return nil, fmt.Errorf("write SQL audit CSV event: %w", err)
|
||||
}
|
||||
}
|
||||
writer.Flush()
|
||||
if err := writer.Error(); err != nil {
|
||||
return nil, fmt.Errorf("flush SQL audit CSV export: %w", err)
|
||||
}
|
||||
return output.Bytes(), nil
|
||||
}
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedExportFormat, format)
|
||||
}
|
||||
|
||||
func (s *Store) loadExportEvents(filter Filter, recordLimit int64, byteLimit int) ([]Event, error) {
|
||||
where, args := buildFilterWhere(filter)
|
||||
var total int64
|
||||
if err := s.db.QueryRowContext(context.Background(),
|
||||
`SELECT COUNT(*) FROM sql_audit_events`+where, args...,
|
||||
).Scan(&total); err != nil {
|
||||
return nil, fmt.Errorf("count SQL audit export events: %w", err)
|
||||
}
|
||||
if recordLimit > 0 && total > recordLimit {
|
||||
return nil, fmt.Errorf("%w: total=%d limit=%d", ErrExportRecordLimit, total, recordLimit)
|
||||
}
|
||||
rows, err := s.db.QueryContext(context.Background(), selectEventColumns+where+
|
||||
` ORDER BY timestamp DESC, sequence DESC`, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query SQL audit export: %w", err)
|
||||
}
|
||||
events := make([]Event, 0, int(total))
|
||||
retainedStringBytes := 0
|
||||
for rows.Next() {
|
||||
if recordLimit > 0 && int64(len(events)) >= recordLimit {
|
||||
_ = rows.Close()
|
||||
return nil, fmt.Errorf("%w: limit=%d", ErrExportRecordLimit, recordLimit)
|
||||
}
|
||||
event, scanErr := scanEvent(rows)
|
||||
if scanErr != nil {
|
||||
_ = rows.Close()
|
||||
return nil, scanErr
|
||||
}
|
||||
retainedStringBytes += eventStringBytes(event)
|
||||
if byteLimit > 0 && retainedStringBytes > byteLimit {
|
||||
_ = rows.Close()
|
||||
return nil, fmt.Errorf("%w: limit=%d", ErrExportSizeLimit, byteLimit)
|
||||
}
|
||||
events = append(events, event)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
_ = rows.Close()
|
||||
return nil, fmt.Errorf("iterate SQL audit export: %w", err)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, fmt.Errorf("close SQL audit export rows: %w", err)
|
||||
}
|
||||
return events, nil
|
||||
}
|
||||
|
||||
func eventStringBytes(event Event) int {
|
||||
return len(event.ID) + len(event.EventType) + len(event.Status) + len(event.ConnectionID) +
|
||||
len(event.ConnectionFingerprint) + len(event.DBType) + len(event.Database) + len(event.QueryID) +
|
||||
len(event.TransactionID) + len(event.Source) + len(event.BoundaryMode) + len(event.CommitMode) +
|
||||
len(event.SQLText) + len(event.SQLFingerprint) + len(event.Error) + len(event.PrevHash) + len(event.Hash)
|
||||
}
|
||||
|
||||
type boundedBuffer struct {
|
||||
buffer bytes.Buffer
|
||||
limit int
|
||||
}
|
||||
|
||||
func newBoundedBuffer(limit int) *boundedBuffer {
|
||||
return &boundedBuffer{limit: limit}
|
||||
}
|
||||
|
||||
func (b *boundedBuffer) Write(payload []byte) (int, error) {
|
||||
if b.limit > 0 && b.buffer.Len()+len(payload) > b.limit {
|
||||
return 0, fmt.Errorf("%w: limit=%d", ErrExportSizeLimit, b.limit)
|
||||
}
|
||||
return b.buffer.Write(payload)
|
||||
}
|
||||
|
||||
func (b *boundedBuffer) Bytes() []byte {
|
||||
return b.buffer.Bytes()
|
||||
}
|
||||
|
||||
var exportCSVHeader = []string{
|
||||
"id", "sequence", "timestamp", "eventType", "status", "connectionId",
|
||||
"connectionFingerprint", "dbType", "database", "queryId", "transactionId",
|
||||
"source", "boundaryMode", "commitMode", "sqlText", "sqlRedacted",
|
||||
"sqlFingerprint", "statementIndex", "statementCount", "durationMs",
|
||||
"rowsAffected", "rowsReturned", "error", "prevHash", "hash",
|
||||
}
|
||||
|
||||
func eventCSVRecord(event Event) []string {
|
||||
return protectCSVRecord([]string{
|
||||
event.ID,
|
||||
strconv.FormatInt(event.Sequence, 10),
|
||||
strconv.FormatInt(event.Timestamp, 10),
|
||||
event.EventType,
|
||||
event.Status,
|
||||
event.ConnectionID,
|
||||
event.ConnectionFingerprint,
|
||||
event.DBType,
|
||||
event.Database,
|
||||
event.QueryID,
|
||||
event.TransactionID,
|
||||
event.Source,
|
||||
event.BoundaryMode,
|
||||
event.CommitMode,
|
||||
event.SQLText,
|
||||
strconv.FormatBool(event.SQLRedacted),
|
||||
event.SQLFingerprint,
|
||||
strconv.Itoa(event.StatementIndex),
|
||||
strconv.Itoa(event.StatementCount),
|
||||
strconv.FormatInt(event.DurationMs, 10),
|
||||
strconv.FormatInt(event.RowsAffected, 10),
|
||||
strconv.FormatInt(event.RowsReturned, 10),
|
||||
event.Error,
|
||||
event.PrevHash,
|
||||
event.Hash,
|
||||
})
|
||||
}
|
||||
|
||||
func protectCSVRecord(record []string) []string {
|
||||
protected := make([]string, len(record))
|
||||
for index, cell := range record {
|
||||
protected[index] = protectCSVFormula(cell)
|
||||
}
|
||||
return protected
|
||||
}
|
||||
|
||||
func protectCSVFormula(value string) string {
|
||||
trimmed := strings.TrimLeft(value, " \t\r\n")
|
||||
if trimmed == "" {
|
||||
return value
|
||||
}
|
||||
switch trimmed[0] {
|
||||
case '=', '+', '-', '@':
|
||||
return "'" + value
|
||||
default:
|
||||
return value
|
||||
}
|
||||
}
|
||||
156
internal/sqlaudit/export_test.go
Normal file
156
internal/sqlaudit/export_test.go
Normal file
@@ -0,0 +1,156 @@
|
||||
package sqlaudit
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestBuildExportIgnoresPaginationAndFiltersAllMatches(t *testing.T) {
|
||||
store := openTestStore(t)
|
||||
now := time.Now().UnixMilli()
|
||||
for index := 0; index < 3; index++ {
|
||||
event := sampleEvent("export-"+string(rune('a'+index)), now+int64(index))
|
||||
event.ConnectionID = "export-connection"
|
||||
if err := store.Append(event); err != nil {
|
||||
t.Fatalf("Append returned error: %v", err)
|
||||
}
|
||||
}
|
||||
other := sampleEvent("other", now+10)
|
||||
other.ConnectionID = "other-connection"
|
||||
if err := store.Append(other); err != nil {
|
||||
t.Fatalf("Append other returned error: %v", err)
|
||||
}
|
||||
|
||||
payload, err := store.BuildExport(Filter{
|
||||
ConnectionID: "export-connection",
|
||||
Page: 3,
|
||||
PageSize: 1,
|
||||
}, "json")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildExport(json) returned error: %v", err)
|
||||
}
|
||||
var events []Event
|
||||
if err := json.Unmarshal(payload, &events); err != nil {
|
||||
t.Fatalf("decode JSON export: %v\npayload=%s", err, payload)
|
||||
}
|
||||
if len(events) != 3 {
|
||||
t.Fatalf("export should ignore pagination and include 3 filtered events, got %d", len(events))
|
||||
}
|
||||
for _, event := range events {
|
||||
if event.ConnectionID != "export-connection" {
|
||||
t.Fatalf("export ignored non-pagination filter: %#v", event)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildCSVExportPreventsFormulaInjection(t *testing.T) {
|
||||
store := openTestStore(t)
|
||||
event := sampleEvent("csv-formula", time.Now().UnixMilli())
|
||||
event.ConnectionID = "=HYPERLINK(\"https://example.test\")"
|
||||
event.Database = " @SUM(1+1)"
|
||||
if err := store.Append(event); err != nil {
|
||||
t.Fatalf("Append returned error: %v", err)
|
||||
}
|
||||
payload, err := store.BuildExport(Filter{}, "CSV")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildExport(csv) returned error: %v", err)
|
||||
}
|
||||
reader := csv.NewReader(bytes.NewReader(payload))
|
||||
records, err := reader.ReadAll()
|
||||
if err != nil {
|
||||
t.Fatalf("read CSV export: %v\npayload=%s", err, payload)
|
||||
}
|
||||
if len(records) != 2 {
|
||||
t.Fatalf("CSV export rows=%d, want header + event", len(records))
|
||||
}
|
||||
connectionColumn := csvHeaderIndex(t, records[0], "connectionId")
|
||||
databaseColumn := csvHeaderIndex(t, records[0], "database")
|
||||
if !strings.HasPrefix(records[1][connectionColumn], "'=") {
|
||||
t.Fatalf("connection ID formula was not escaped: %q", records[1][connectionColumn])
|
||||
}
|
||||
if !strings.HasPrefix(records[1][databaseColumn], "'@") {
|
||||
t.Fatalf("whitespace-prefixed formula was not escaped: %q", records[1][databaseColumn])
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildExportRejectsUnsupportedFormat(t *testing.T) {
|
||||
store := openTestStore(t)
|
||||
_, err := store.BuildExport(Filter{}, "xlsx")
|
||||
if !errors.Is(err, ErrUnsupportedExportFormat) {
|
||||
t.Fatalf("BuildExport error=%v, want ErrUnsupportedExportFormat", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildExportReturnsRecognizableRecordLimitError(t *testing.T) {
|
||||
store := openTestStore(t)
|
||||
now := time.Now().UnixMilli()
|
||||
if err := store.Append(sampleEvent("limit-a", now)); err != nil {
|
||||
t.Fatalf("Append first returned error: %v", err)
|
||||
}
|
||||
if err := store.Append(sampleEvent("limit-b", now+1)); err != nil {
|
||||
t.Fatalf("Append second returned error: %v", err)
|
||||
}
|
||||
for _, format := range []string{"json", "csv"} {
|
||||
if _, err := store.buildExportWithLimits(Filter{}, format, 1, maxExportBytes); !errors.Is(err, ErrExportRecordLimit) {
|
||||
t.Fatalf("%s export error=%v, want ErrExportRecordLimit", format, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildExportReturnsRecognizableSizeLimitError(t *testing.T) {
|
||||
store := openTestStore(t)
|
||||
event := sampleEvent("large-export", time.Now().UnixMilli())
|
||||
event.Database = strings.Repeat("database", 30)
|
||||
if err := store.Append(event); err != nil {
|
||||
t.Fatalf("Append returned error: %v", err)
|
||||
}
|
||||
for _, format := range []string{"json", "csv"} {
|
||||
if _, err := store.buildExportWithLimits(Filter{}, format, maxExportRecords, 64); !errors.Is(err, ErrExportSizeLimit) {
|
||||
t.Fatalf("%s export error=%v, want ErrExportSizeLimit", format, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadExportEventsClosesRowsBeforeReturning(t *testing.T) {
|
||||
store := openTestStore(t)
|
||||
now := time.Now().UnixMilli()
|
||||
if err := store.Append(sampleEvent("export-connection-release", now)); err != nil {
|
||||
t.Fatalf("Append returned error: %v", err)
|
||||
}
|
||||
events, err := store.loadExportEvents(normalizeFilter(Filter{}), maxExportRecords, maxExportBytes)
|
||||
if err != nil || len(events) != 1 {
|
||||
t.Fatalf("loadExportEvents events=%#v err=%v", events, err)
|
||||
}
|
||||
if inUse := store.db.Stats().InUse; inUse != 0 {
|
||||
t.Fatalf("loadExportEvents returned before releasing SQLite rows: inUse=%d", inUse)
|
||||
}
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- store.Append(sampleEvent("append-after-export-load", now+1))
|
||||
}()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatalf("Append after export load returned error: %v", err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("export row cursor retained the store's only SQLite connection")
|
||||
}
|
||||
}
|
||||
|
||||
func csvHeaderIndex(t *testing.T, header []string, name string) int {
|
||||
t.Helper()
|
||||
for index, value := range header {
|
||||
if value == name {
|
||||
return index
|
||||
}
|
||||
}
|
||||
t.Fatalf("CSV header %q missing from %#v", name, header)
|
||||
return -1
|
||||
}
|
||||
800
internal/sqlaudit/redact.go
Normal file
800
internal/sqlaudit/redact.go
Normal file
@@ -0,0 +1,800 @@
|
||||
package sqlaudit
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
var sqlQueryTypes = map[string]struct{}{
|
||||
"mysql": {}, "mariadb": {}, "goldendb": {}, "greatdb": {}, "gdb": {},
|
||||
"oceanbase": {}, "doris": {}, "diros": {}, "starrocks": {}, "sphinx": {},
|
||||
"postgres": {}, "postgresql": {}, "sqlserver": {}, "mssql": {}, "sqlite": {},
|
||||
"duckdb": {}, "oracle": {}, "dameng": {}, "dm": {}, "kingbase": {}, "highgo": {},
|
||||
"vastbase": {}, "opengauss": {}, "gaussdb": {}, "iris": {}, "intersystems": {},
|
||||
"tdengine": {}, "iotdb": {}, "clickhouse": {}, "trino": {}, "custom": {},
|
||||
"gonavi": {},
|
||||
}
|
||||
|
||||
var redisReadCommands = map[string]struct{}{
|
||||
"GET": {}, "MGET": {}, "GETDEL": {}, "GETEX": {}, "STRLEN": {}, "TYPE": {},
|
||||
"TTL": {}, "PTTL": {}, "EXPIRETIME": {}, "PEXPIRETIME": {}, "EXISTS": {},
|
||||
"DEL": {}, "UNLINK": {}, "TOUCH": {}, "DUMP": {}, "OBJECT": {}, "KEYS": {},
|
||||
"SCAN": {}, "HGET": {}, "HMGET": {}, "HGETALL": {}, "HEXISTS": {}, "HLEN": {},
|
||||
"HKEYS": {}, "HVALS": {}, "HSCAN": {}, "LINDEX": {}, "LLEN": {}, "LRANGE": {},
|
||||
"LPOS": {}, "SCARD": {}, "SMEMBERS": {}, "SISMEMBER": {}, "SMISMEMBER": {},
|
||||
"SSCAN": {}, "ZCARD": {}, "ZCOUNT": {}, "ZLEXCOUNT": {}, "ZRANGE": {},
|
||||
"ZRANGEBYSCORE": {}, "ZRANK": {}, "ZREVRANK": {}, "ZSCORE": {}, "ZMSCORE": {},
|
||||
"ZSCAN": {}, "XRANGE": {}, "XREVRANGE": {}, "XLEN": {}, "XREAD": {}, "XINFO": {},
|
||||
"PFCOUNT": {}, "BITCOUNT": {}, "GETBIT": {}, "GEODIST": {}, "GEOHASH": {},
|
||||
"GEOPOS": {}, "GEOSEARCH": {}, "JSON.GET": {}, "JSON.MGET": {}, "JSON.TYPE": {},
|
||||
"JSON.OBJKEYS": {}, "JSON.OBJLEN": {}, "JSON.ARRLEN": {}, "TS.GET": {},
|
||||
"TS.RANGE": {}, "TS.REVRANGE": {}, "TS.MGET": {}, "TS.MRANGE": {},
|
||||
}
|
||||
|
||||
var redisMultiKeyCommands = map[string]struct{}{
|
||||
"MGET": {}, "DEL": {}, "UNLINK": {}, "EXISTS": {}, "TOUCH": {},
|
||||
}
|
||||
|
||||
// RedactQuery dispatches query sanitization by data-source type. SQL dialects
|
||||
// use the SQL lexer, Redis retains only commands/keys and replaces values, and
|
||||
// known non-SQL/message or unknown query languages default to metadata-only.
|
||||
func RedactQuery(dbType, text string) string {
|
||||
normalizedType := normalizeQueryType(dbType)
|
||||
if normalizedType == "redis" {
|
||||
return redactRedisQuery(text)
|
||||
}
|
||||
if _, ok := sqlQueryTypes[normalizedType]; ok {
|
||||
return RedactSQL(text)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
const (
|
||||
maxSQLRunes = 64 * 1024
|
||||
maxErrorRunes = 4 * 1024
|
||||
)
|
||||
|
||||
var (
|
||||
sensitiveAssignmentPattern = regexp.MustCompile(`(?i)\b(password|passwd|pwd|token|secret|api[_-]?key|access[_-]?key)\b(\s*(?:=|:)\s*)([^\s,;]+)`)
|
||||
identifiedByPattern = regexp.MustCompile(`(?i)\b(identified\s+by)(\s+)([^\s,;]+)`)
|
||||
uriUserInfoPattern = regexp.MustCompile(`(?i)([a-z][a-z0-9+.-]*://)([^\s/@:]+)(?::[^\s/@]*)?@`)
|
||||
bareUserInfoPattern = regexp.MustCompile(`(?i)\b[^\s:@/]+:[^\s@/]+@(\[[^\]\s]+\]|[a-z0-9.-]+)`)
|
||||
slashUserInfoPattern = regexp.MustCompile(`(?i)\b[^\s/@:]+/[^\s@/]+@([a-z0-9.-]+)(?:/[^\s,;]*)?`)
|
||||
bearerPattern = regexp.MustCompile(`(?i)\bbearer\s+[a-z0-9._~+/=-]+`)
|
||||
basicAuthPattern = regexp.MustCompile(`(?i)\bbasic\s+[a-z0-9+/=]+`)
|
||||
postgresKeyDetailPattern = regexp.MustCompile(`(?i)(\bkey\s*\([^\)\r\n]*\)\s*=\s*)\([^\)\r\n]*\)`)
|
||||
duplicateKeyValuePattern = regexp.MustCompile(`(?i)(\bduplicate\s+key\s+value\s+(?:is|was)\s*)\([^\)\r\n]*\)`)
|
||||
failingRowPattern = regexp.MustCompile(`(?i)(\bfailing\s+row\s+contains\s*)\([^\)\r\n]*\)`)
|
||||
)
|
||||
|
||||
// RedactSQL removes comments and literal values while retaining enough SQL
|
||||
// structure for local diagnostics. It intentionally favors confidentiality
|
||||
// over perfectly preserving dialect-specific quoted identifiers.
|
||||
func RedactSQL(sqlText string) string {
|
||||
if strings.TrimSpace(sqlText) == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
var out strings.Builder
|
||||
out.Grow(len(sqlText))
|
||||
for index := 0; index < len(sqlText); {
|
||||
if hasPrefixAt(sqlText, index, "--") || sqlText[index] == '#' {
|
||||
index = skipLineComment(sqlText, index)
|
||||
if index < len(sqlText) && (sqlText[index] == '\r' || sqlText[index] == '\n') {
|
||||
out.WriteByte('\n')
|
||||
}
|
||||
continue
|
||||
}
|
||||
if hasPrefixAt(sqlText, index, "/*") {
|
||||
index = skipBlockComment(sqlText, index+2)
|
||||
out.WriteByte(' ')
|
||||
continue
|
||||
}
|
||||
if next, ok := skipOracleQuotedLiteral(sqlText, index); ok {
|
||||
out.WriteByte('?')
|
||||
index = next
|
||||
continue
|
||||
}
|
||||
if sqlText[index] == '$' {
|
||||
if next, ok := skipDollarQuotedLiteral(sqlText, index); ok {
|
||||
out.WriteByte('?')
|
||||
index = next
|
||||
continue
|
||||
}
|
||||
}
|
||||
if sqlText[index] == '\'' || sqlText[index] == '"' {
|
||||
quote := sqlText[index]
|
||||
out.WriteByte(quote)
|
||||
out.WriteByte('?')
|
||||
out.WriteByte(quote)
|
||||
index = skipQuoted(sqlText, index, quote)
|
||||
continue
|
||||
}
|
||||
if isNumberStart(sqlText, index) {
|
||||
out.WriteByte('?')
|
||||
index = skipNumber(sqlText, index)
|
||||
continue
|
||||
}
|
||||
|
||||
r, size := utf8.DecodeRuneInString(sqlText[index:])
|
||||
if r == utf8.RuneError && size == 1 {
|
||||
out.WriteRune(unicode.ReplacementChar)
|
||||
index++
|
||||
continue
|
||||
}
|
||||
if unicode.IsControl(r) && r != '\n' && r != '\r' && r != '\t' {
|
||||
out.WriteByte(' ')
|
||||
} else {
|
||||
out.WriteRune(r)
|
||||
}
|
||||
index += size
|
||||
}
|
||||
|
||||
redacted := sensitiveAssignmentPattern.ReplaceAllString(out.String(), `${1}${2}?`)
|
||||
redacted = identifiedByPattern.ReplaceAllString(redacted, `${1}${2}?`)
|
||||
redacted = uriUserInfoPattern.ReplaceAllString(redacted, `${1}***:***@`)
|
||||
redacted = bearerPattern.ReplaceAllString(redacted, "Bearer ***")
|
||||
return truncateRunes(strings.TrimSpace(redacted), maxSQLRunes)
|
||||
}
|
||||
|
||||
// RedactError removes common credential forms and quoted values from a driver
|
||||
// error before it enters the audit database.
|
||||
func RedactError(message string) string {
|
||||
message = strings.TrimSpace(message)
|
||||
if message == "" {
|
||||
return ""
|
||||
}
|
||||
message = uriUserInfoPattern.ReplaceAllString(message, `${1}***:***@`)
|
||||
message = bareUserInfoPattern.ReplaceAllString(message, `***:***@$1`)
|
||||
message = slashUserInfoPattern.ReplaceAllString(message, `***/***@$1`)
|
||||
message = bearerPattern.ReplaceAllString(message, "Bearer ***")
|
||||
message = basicAuthPattern.ReplaceAllString(message, "Basic ***")
|
||||
message = postgresKeyDetailPattern.ReplaceAllString(message, `${1}(?)`)
|
||||
message = duplicateKeyValuePattern.ReplaceAllString(message, `${1}(?)`)
|
||||
message = failingRowPattern.ReplaceAllString(message, `${1}(?)`)
|
||||
message = sensitiveAssignmentPattern.ReplaceAllString(message, `${1}${2}***`)
|
||||
message = identifiedByPattern.ReplaceAllString(message, `${1}${2}***`)
|
||||
message = redactQuotedSegments(message)
|
||||
message = strings.Map(func(r rune) rune {
|
||||
if unicode.IsControl(r) {
|
||||
return ' '
|
||||
}
|
||||
return r
|
||||
}, message)
|
||||
return truncateRunes(strings.TrimSpace(message), maxErrorRunes)
|
||||
}
|
||||
|
||||
func normalizeQueryType(dbType string) string {
|
||||
normalized := strings.ToLower(strings.TrimSpace(dbType))
|
||||
switch normalized {
|
||||
case "redis-cluster", "redis_cluster", "redis-sentinel", "redis_sentinel":
|
||||
return "redis"
|
||||
case "rocket-mq", "rocket_mq", "apache-rocketmq", "apache_rocketmq", "rmq":
|
||||
return "rocketmq"
|
||||
case "apache-kafka", "apache_kafka":
|
||||
return "kafka"
|
||||
case "rabbit-mq", "rabbit_mq":
|
||||
return "rabbitmq"
|
||||
case "elastic":
|
||||
return "elasticsearch"
|
||||
case "chromadb", "chroma-db":
|
||||
return "chroma"
|
||||
case "qdrantdb", "qdrant-db":
|
||||
return "qdrant"
|
||||
case "milvusdb", "milvus-db":
|
||||
return "milvus"
|
||||
default:
|
||||
return normalized
|
||||
}
|
||||
}
|
||||
|
||||
func redactRedisQuery(text string) string {
|
||||
tokens, ok := tokenizeRedisQuery(text)
|
||||
if !ok || len(tokens) == 0 {
|
||||
return ""
|
||||
}
|
||||
command := strings.ToUpper(tokens[0])
|
||||
switch command {
|
||||
case "AUTH":
|
||||
if len(tokens) >= 3 {
|
||||
return "AUTH ? ?"
|
||||
}
|
||||
if len(tokens) == 2 {
|
||||
return "AUTH ?"
|
||||
}
|
||||
return "AUTH"
|
||||
case "HELLO":
|
||||
return redactRedisHello(tokens)
|
||||
case "CONFIG":
|
||||
return redactRedisConfig(tokens)
|
||||
case "ACL":
|
||||
return redactRedisACL(tokens)
|
||||
case "CLIENT":
|
||||
return redactRedisClient(tokens)
|
||||
case "SCRIPT":
|
||||
return redactRedisScript(tokens)
|
||||
case "FUNCTION":
|
||||
return redactRedisFunction(tokens)
|
||||
case "PING", "ECHO":
|
||||
if len(tokens) > 1 {
|
||||
return command + " ?"
|
||||
}
|
||||
return command
|
||||
case "MSET", "MSETNX":
|
||||
return redactRedisAlternatingPairs(command, tokens, 1)
|
||||
case "HSET", "HMSET":
|
||||
return redactRedisHashPairs(command, tokens)
|
||||
case "SET", "SETNX", "GETSET", "APPEND", "SETRANGE", "INCRBY", "INCRBYFLOAT",
|
||||
"DECRBY", "SETBIT", "SETEX", "PSETEX", "RESTORE", "LSET", "LINSERT",
|
||||
"HSETNX", "HINCRBY", "HINCRBYFLOAT", "LPUSH", "LPUSHX", "RPUSH", "RPUSHX",
|
||||
"LREM", "SADD", "SREM", "SMOVE",
|
||||
"ZADD", "ZINCRBY", "ZREM", "GEOADD", "PFADD", "PUBLISH", "SPUBLISH",
|
||||
"XADD", "XACK", "XCLAIM", "XAUTOCLAIM", "XDEL", "XGROUP", "XTRIM",
|
||||
"JSON.SET", "JSON.MERGE", "JSON.ARRAPPEND", "JSON.ARRINSERT", "JSON.ARRTRIM",
|
||||
"JSON.NUMINCRBY", "JSON.NUMMULTBY", "JSON.STRAPPEND", "TS.ADD", "TS.INCRBY",
|
||||
"TS.DECRBY":
|
||||
return redactRedisKeyAndPayload(command, tokens)
|
||||
case "EVAL", "EVALSHA", "FCALL", "FCALL_RO":
|
||||
return redactRedisScriptInvocation(command, tokens)
|
||||
case "MIGRATE", "MODULE":
|
||||
return command + " ?"
|
||||
case "SELECT", "SWAPDB":
|
||||
return command + redactRedisSafeArguments(tokens[1:])
|
||||
default:
|
||||
if _, ok := redisReadCommands[command]; !ok {
|
||||
return ""
|
||||
}
|
||||
if len(tokens) == 1 {
|
||||
return command
|
||||
}
|
||||
if _, ok := redisMultiKeyCommands[command]; ok {
|
||||
return command + redactRedisSafeArguments(tokens[1:])
|
||||
}
|
||||
// Retain the first key/pattern only. Later arguments can be members,
|
||||
// payloads or dialect-specific options and are omitted conservatively.
|
||||
return command + " " + formatRedisToken(tokens[1])
|
||||
}
|
||||
}
|
||||
|
||||
func redactRedisHello(tokens []string) string {
|
||||
parts := []string{"HELLO"}
|
||||
index := 1
|
||||
if index < len(tokens) {
|
||||
if _, err := strconv.Atoi(tokens[index]); err == nil {
|
||||
parts = append(parts, tokens[index])
|
||||
} else {
|
||||
parts = append(parts, "?")
|
||||
}
|
||||
index++
|
||||
}
|
||||
for index < len(tokens) {
|
||||
switch strings.ToUpper(tokens[index]) {
|
||||
case "AUTH":
|
||||
parts = append(parts, "AUTH", "?", "?")
|
||||
index += 3
|
||||
case "SETNAME":
|
||||
parts = append(parts, "SETNAME", "?")
|
||||
index += 2
|
||||
default:
|
||||
// Unknown HELLO options may carry data; retain no further text.
|
||||
index = len(tokens)
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
func redactRedisConfig(tokens []string) string {
|
||||
if len(tokens) < 2 {
|
||||
return "CONFIG"
|
||||
}
|
||||
subcommand := strings.ToUpper(tokens[1])
|
||||
switch subcommand {
|
||||
case "SET":
|
||||
if len(tokens) >= 3 {
|
||||
return "CONFIG SET " + formatRedisToken(tokens[2]) + " ?"
|
||||
}
|
||||
return "CONFIG SET ?"
|
||||
case "GET":
|
||||
if len(tokens) >= 3 {
|
||||
return "CONFIG GET " + formatRedisToken(tokens[2])
|
||||
}
|
||||
return "CONFIG GET"
|
||||
case "RESETSTAT", "REWRITE":
|
||||
return "CONFIG " + subcommand
|
||||
default:
|
||||
return "CONFIG"
|
||||
}
|
||||
}
|
||||
|
||||
func redactRedisACL(tokens []string) string {
|
||||
if len(tokens) < 2 {
|
||||
return "ACL"
|
||||
}
|
||||
subcommand := strings.ToUpper(tokens[1])
|
||||
switch subcommand {
|
||||
case "SETUSER":
|
||||
if len(tokens) >= 3 {
|
||||
return "ACL SETUSER " + formatRedisToken(tokens[2]) + " ?"
|
||||
}
|
||||
return "ACL SETUSER ?"
|
||||
case "GETUSER", "DELUSER":
|
||||
if len(tokens) >= 3 {
|
||||
return "ACL " + subcommand + " " + formatRedisToken(tokens[2])
|
||||
}
|
||||
return "ACL " + subcommand
|
||||
case "LIST", "USERS", "WHOAMI", "CAT", "GENPASS", "LOG", "SAVE":
|
||||
return "ACL " + subcommand
|
||||
default:
|
||||
return "ACL"
|
||||
}
|
||||
}
|
||||
|
||||
func redactRedisClient(tokens []string) string {
|
||||
if len(tokens) < 2 {
|
||||
return "CLIENT"
|
||||
}
|
||||
subcommand := strings.ToUpper(tokens[1])
|
||||
switch subcommand {
|
||||
case "SETNAME", "SETINFO", "KILL", "UNBLOCK":
|
||||
return "CLIENT " + subcommand + " ?"
|
||||
case "GETNAME", "ID", "INFO", "LIST", "PAUSE", "UNPAUSE", "REPLY", "TRACKING", "CACHING":
|
||||
return "CLIENT " + subcommand
|
||||
default:
|
||||
return "CLIENT"
|
||||
}
|
||||
}
|
||||
|
||||
func redactRedisScript(tokens []string) string {
|
||||
if len(tokens) < 2 {
|
||||
return "SCRIPT"
|
||||
}
|
||||
subcommand := strings.ToUpper(tokens[1])
|
||||
switch subcommand {
|
||||
case "LOAD", "DEBUG":
|
||||
return "SCRIPT " + subcommand + " ?"
|
||||
case "EXISTS", "FLUSH", "KILL", "HELP":
|
||||
return "SCRIPT " + subcommand
|
||||
default:
|
||||
return "SCRIPT"
|
||||
}
|
||||
}
|
||||
|
||||
func redactRedisFunction(tokens []string) string {
|
||||
if len(tokens) < 2 {
|
||||
return "FUNCTION"
|
||||
}
|
||||
subcommand := strings.ToUpper(tokens[1])
|
||||
switch subcommand {
|
||||
case "LOAD", "RESTORE":
|
||||
return "FUNCTION " + subcommand + " ?"
|
||||
case "DELETE":
|
||||
if len(tokens) >= 3 {
|
||||
return "FUNCTION DELETE " + formatRedisToken(tokens[2])
|
||||
}
|
||||
return "FUNCTION DELETE"
|
||||
case "DUMP", "FLUSH", "KILL", "LIST", "STATS", "HELP":
|
||||
return "FUNCTION " + subcommand
|
||||
default:
|
||||
return "FUNCTION"
|
||||
}
|
||||
}
|
||||
|
||||
func redactRedisAlternatingPairs(command string, tokens []string, start int) string {
|
||||
if len(tokens) <= start || (len(tokens)-start)%2 != 0 {
|
||||
return command + " ?"
|
||||
}
|
||||
parts := []string{command}
|
||||
for index := start; index < len(tokens); index += 2 {
|
||||
parts = append(parts, formatRedisToken(tokens[index]))
|
||||
if index+1 < len(tokens) {
|
||||
parts = append(parts, "?")
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
func redactRedisHashPairs(command string, tokens []string) string {
|
||||
if len(tokens) < 2 {
|
||||
return command
|
||||
}
|
||||
if (len(tokens)-2)%2 != 0 {
|
||||
return command + " " + formatRedisToken(tokens[1]) + " ?"
|
||||
}
|
||||
parts := []string{command, formatRedisToken(tokens[1])}
|
||||
for index := 2; index < len(tokens); index += 2 {
|
||||
parts = append(parts, formatRedisToken(tokens[index]))
|
||||
if index+1 < len(tokens) {
|
||||
parts = append(parts, "?")
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
func redactRedisKeyAndPayload(command string, tokens []string) string {
|
||||
if len(tokens) < 2 {
|
||||
return command
|
||||
}
|
||||
result := command + " " + formatRedisToken(tokens[1])
|
||||
if len(tokens) > 2 {
|
||||
result += " ?"
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func redactRedisScriptInvocation(command string, tokens []string) string {
|
||||
if len(tokens) < 3 {
|
||||
return command + " ?"
|
||||
}
|
||||
numKeys, err := strconv.Atoi(tokens[2])
|
||||
if err != nil || numKeys < 0 {
|
||||
return command + " ?"
|
||||
}
|
||||
parts := []string{command, "?", strconv.Itoa(numKeys)}
|
||||
for index := 0; index < numKeys && 3+index < len(tokens); index++ {
|
||||
parts = append(parts, formatRedisToken(tokens[3+index]))
|
||||
}
|
||||
if 3+numKeys < len(tokens) {
|
||||
parts = append(parts, "?")
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
func redactRedisSafeArguments(tokens []string) string {
|
||||
if len(tokens) == 0 {
|
||||
return ""
|
||||
}
|
||||
parts := make([]string, 0, len(tokens))
|
||||
for _, token := range tokens {
|
||||
parts = append(parts, formatRedisToken(token))
|
||||
}
|
||||
return " " + strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
func tokenizeRedisQuery(text string) ([]string, bool) {
|
||||
text = strings.TrimSpace(text)
|
||||
if text == "" {
|
||||
return nil, true
|
||||
}
|
||||
var tokens []string
|
||||
var current strings.Builder
|
||||
var quote rune
|
||||
escaped := false
|
||||
tokenStarted := false
|
||||
flush := func() {
|
||||
if !tokenStarted {
|
||||
return
|
||||
}
|
||||
tokens = append(tokens, current.String())
|
||||
current.Reset()
|
||||
tokenStarted = false
|
||||
}
|
||||
for _, r := range text {
|
||||
if escaped {
|
||||
current.WriteRune(r)
|
||||
escaped = false
|
||||
tokenStarted = true
|
||||
continue
|
||||
}
|
||||
if r == '\\' {
|
||||
escaped = true
|
||||
tokenStarted = true
|
||||
continue
|
||||
}
|
||||
if quote != 0 {
|
||||
if r == quote {
|
||||
quote = 0
|
||||
} else {
|
||||
current.WriteRune(r)
|
||||
}
|
||||
tokenStarted = true
|
||||
continue
|
||||
}
|
||||
if r == '\'' || r == '"' {
|
||||
quote = r
|
||||
tokenStarted = true
|
||||
continue
|
||||
}
|
||||
if unicode.IsSpace(r) {
|
||||
flush()
|
||||
continue
|
||||
}
|
||||
current.WriteRune(r)
|
||||
tokenStarted = true
|
||||
}
|
||||
if escaped {
|
||||
current.WriteRune('\\')
|
||||
}
|
||||
if quote != 0 {
|
||||
return nil, false
|
||||
}
|
||||
flush()
|
||||
return tokens, true
|
||||
}
|
||||
|
||||
func formatRedisToken(token string) string {
|
||||
token = truncateRunes(strings.Map(func(r rune) rune {
|
||||
if unicode.IsControl(r) {
|
||||
return ' '
|
||||
}
|
||||
return r
|
||||
}, token), 512)
|
||||
if token == "" {
|
||||
return `""`
|
||||
}
|
||||
if strings.IndexFunc(token, unicode.IsSpace) >= 0 {
|
||||
return strconv.Quote(token)
|
||||
}
|
||||
return token
|
||||
}
|
||||
|
||||
func queryFingerprintSeed(dbType, redactedText string) string {
|
||||
if redactedText != "" {
|
||||
return redactedText
|
||||
}
|
||||
// Metadata-only query languages intentionally do not include raw input in
|
||||
// their fingerprint. Payload-derived hashes can be dictionary-guessed when
|
||||
// messages contain low-entropy tokens, card numbers or common credentials.
|
||||
return "metadata:" + normalizeQueryType(dbType)
|
||||
}
|
||||
|
||||
func sanitizeEvent(event Event, settings Settings) Event {
|
||||
event.Sequence = 0
|
||||
event.PrevHash = ""
|
||||
event.Hash = ""
|
||||
event.EventType = strings.ToLower(sanitizeLabel(event.EventType, 64))
|
||||
event.Status = strings.ToLower(sanitizeLabel(event.Status, 64))
|
||||
event.ConnectionID = sanitizeLabel(event.ConnectionID, 256)
|
||||
event.DBType = strings.ToLower(sanitizeLabel(event.DBType, 64))
|
||||
event.Database = sanitizeLabel(event.Database, 512)
|
||||
event.QueryID = sanitizeLabel(event.QueryID, 256)
|
||||
event.TransactionID = sanitizeLabel(event.TransactionID, 256)
|
||||
event.Source = strings.ToLower(sanitizeLabel(event.Source, 64))
|
||||
event.BoundaryMode = normalizeBoundaryMode(event.BoundaryMode)
|
||||
event.CommitMode = normalizeCommitMode(event.CommitMode)
|
||||
|
||||
rawSQL := event.SQLText
|
||||
redactedSQL := RedactQuery(event.DBType, rawSQL)
|
||||
fingerprintSeed := queryFingerprintSeed(event.DBType, redactedSQL)
|
||||
event.SQLFingerprint = versionedDigest("sqlaudit-sql-fingerprint-v1", fingerprintSeed)
|
||||
event.SQLRedacted = true
|
||||
if settings.CaptureMode == CaptureModeMetadata {
|
||||
event.SQLText = ""
|
||||
} else {
|
||||
event.SQLText = redactedSQL
|
||||
}
|
||||
|
||||
fingerprintSeed = strings.TrimSpace(event.ConnectionFingerprint)
|
||||
if fingerprintSeed == "" {
|
||||
fingerprintSeed = strings.Join([]string{event.ConnectionID, event.DBType, event.Database}, "\x00")
|
||||
}
|
||||
event.ConnectionFingerprint = versionedDigest("sqlaudit-connection-fingerprint-v1", fingerprintSeed)
|
||||
event.Error = RedactError(event.Error)
|
||||
if event.StatementIndex < 0 {
|
||||
event.StatementIndex = 0
|
||||
}
|
||||
if event.StatementCount < 0 {
|
||||
event.StatementCount = 0
|
||||
}
|
||||
if event.DurationMs < 0 {
|
||||
event.DurationMs = 0
|
||||
}
|
||||
if event.RowsAffected < 0 {
|
||||
event.RowsAffected = 0
|
||||
}
|
||||
if event.RowsReturned < 0 {
|
||||
event.RowsReturned = 0
|
||||
}
|
||||
return event
|
||||
}
|
||||
|
||||
func normalizeBoundaryMode(value string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(value)) {
|
||||
case BoundaryModeDriverAPI:
|
||||
return BoundaryModeDriverAPI
|
||||
case BoundaryModeTextSQL:
|
||||
return BoundaryModeTextSQL
|
||||
case BoundaryModeImplicit:
|
||||
return BoundaryModeImplicit
|
||||
default:
|
||||
return BoundaryModeUnknown
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeCommitMode(value string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(value)) {
|
||||
case CommitModeAuto:
|
||||
return CommitModeAuto
|
||||
case CommitModeManual:
|
||||
return CommitModeManual
|
||||
case CommitModePending:
|
||||
return CommitModePending
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func versionedDigest(version, value string) string {
|
||||
digest := sha256.Sum256([]byte(version + "\x00" + value))
|
||||
return hex.EncodeToString(digest[:])
|
||||
}
|
||||
|
||||
func sanitizeLabel(value string, maxRunes int) string {
|
||||
value = strings.Map(func(r rune) rune {
|
||||
if unicode.IsControl(r) {
|
||||
return ' '
|
||||
}
|
||||
return r
|
||||
}, strings.TrimSpace(value))
|
||||
return truncateRunes(value, maxRunes)
|
||||
}
|
||||
|
||||
func truncateRunes(value string, max int) string {
|
||||
if max <= 0 || utf8.RuneCountInString(value) <= max {
|
||||
return value
|
||||
}
|
||||
runes := []rune(value)
|
||||
return string(runes[:max])
|
||||
}
|
||||
|
||||
func redactQuotedSegments(value string) string {
|
||||
var out strings.Builder
|
||||
out.Grow(len(value))
|
||||
for index := 0; index < len(value); {
|
||||
if value[index] == '\'' || value[index] == '"' {
|
||||
quote := value[index]
|
||||
out.WriteByte(quote)
|
||||
out.WriteByte('?')
|
||||
out.WriteByte(quote)
|
||||
index = skipQuoted(value, index, quote)
|
||||
continue
|
||||
}
|
||||
out.WriteByte(value[index])
|
||||
index++
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
func skipQuoted(value string, start int, quote byte) int {
|
||||
for index := start + 1; index < len(value); index++ {
|
||||
if value[index] == '\\' {
|
||||
index++
|
||||
continue
|
||||
}
|
||||
if value[index] != quote {
|
||||
continue
|
||||
}
|
||||
if index+1 < len(value) && value[index+1] == quote {
|
||||
index++
|
||||
continue
|
||||
}
|
||||
return index + 1
|
||||
}
|
||||
return len(value)
|
||||
}
|
||||
|
||||
func skipLineComment(value string, start int) int {
|
||||
for index := start; index < len(value); index++ {
|
||||
if value[index] == '\r' || value[index] == '\n' {
|
||||
return index
|
||||
}
|
||||
}
|
||||
return len(value)
|
||||
}
|
||||
|
||||
func skipBlockComment(value string, start int) int {
|
||||
depth := 1
|
||||
for index := start; index+1 < len(value); {
|
||||
switch value[index : index+2] {
|
||||
case "/*":
|
||||
depth++
|
||||
index += 2
|
||||
case "*/":
|
||||
depth--
|
||||
index += 2
|
||||
if depth == 0 {
|
||||
return index
|
||||
}
|
||||
default:
|
||||
index++
|
||||
}
|
||||
}
|
||||
return len(value)
|
||||
}
|
||||
|
||||
func skipDollarQuotedLiteral(value string, start int) (int, bool) {
|
||||
endTag := start + 1
|
||||
for endTag < len(value) && (value[endTag] == '_' || isASCIILetterOrDigit(value[endTag])) {
|
||||
endTag++
|
||||
}
|
||||
if endTag >= len(value) || value[endTag] != '$' {
|
||||
return start, false
|
||||
}
|
||||
tag := value[start : endTag+1]
|
||||
contentEnd := strings.Index(value[endTag+1:], tag)
|
||||
if contentEnd < 0 {
|
||||
return len(value), true
|
||||
}
|
||||
return endTag + 1 + contentEnd + len(tag), true
|
||||
}
|
||||
|
||||
func skipOracleQuotedLiteral(value string, start int) (int, bool) {
|
||||
if start+3 >= len(value) || (value[start] != 'q' && value[start] != 'Q') || value[start+1] != '\'' {
|
||||
return start, false
|
||||
}
|
||||
opening := value[start+2]
|
||||
closing := opening
|
||||
switch opening {
|
||||
case '[':
|
||||
closing = ']'
|
||||
case '{':
|
||||
closing = '}'
|
||||
case '(':
|
||||
closing = ')'
|
||||
case '<':
|
||||
closing = '>'
|
||||
}
|
||||
for index := start + 3; index+1 < len(value); index++ {
|
||||
if value[index] == closing && value[index+1] == '\'' {
|
||||
return index + 2, true
|
||||
}
|
||||
}
|
||||
return len(value), true
|
||||
}
|
||||
|
||||
func isNumberStart(value string, index int) bool {
|
||||
if index >= len(value) || value[index] < '0' || value[index] > '9' {
|
||||
return false
|
||||
}
|
||||
if index == 0 {
|
||||
return true
|
||||
}
|
||||
previous := value[index-1]
|
||||
return !(previous == '_' || previous == '$' || isASCIILetterOrDigit(previous))
|
||||
}
|
||||
|
||||
func skipNumber(value string, start int) int {
|
||||
index := start
|
||||
if index+2 <= len(value) && index+1 < len(value) && value[index] == '0' && (value[index+1] == 'x' || value[index+1] == 'X') {
|
||||
index += 2
|
||||
for index < len(value) && isASCIIHex(value[index]) {
|
||||
index++
|
||||
}
|
||||
return index
|
||||
}
|
||||
for index < len(value) && value[index] >= '0' && value[index] <= '9' {
|
||||
index++
|
||||
}
|
||||
if index < len(value) && value[index] == '.' {
|
||||
index++
|
||||
for index < len(value) && value[index] >= '0' && value[index] <= '9' {
|
||||
index++
|
||||
}
|
||||
}
|
||||
if index < len(value) && (value[index] == 'e' || value[index] == 'E') {
|
||||
index++
|
||||
if index < len(value) && (value[index] == '+' || value[index] == '-') {
|
||||
index++
|
||||
}
|
||||
for index < len(value) && value[index] >= '0' && value[index] <= '9' {
|
||||
index++
|
||||
}
|
||||
}
|
||||
return index
|
||||
}
|
||||
|
||||
func hasPrefixAt(value string, index int, prefix string) bool {
|
||||
return index >= 0 && index+len(prefix) <= len(value) && value[index:index+len(prefix)] == prefix
|
||||
}
|
||||
|
||||
func isASCIILetterOrDigit(value byte) bool {
|
||||
return value >= 'a' && value <= 'z' || value >= 'A' && value <= 'Z' || value >= '0' && value <= '9'
|
||||
}
|
||||
|
||||
func isASCIIHex(value byte) bool {
|
||||
return value >= '0' && value <= '9' || value >= 'a' && value <= 'f' || value >= 'A' && value <= 'F'
|
||||
}
|
||||
167
internal/sqlaudit/redact_test.go
Normal file
167
internal/sqlaudit/redact_test.go
Normal file
@@ -0,0 +1,167 @@
|
||||
package sqlaudit
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRedactSQLRemovesDialectLiteralsCommentsAndCredentials(t *testing.T) {
|
||||
input := `-- comment-secret
|
||||
SELECT 'alice-secret', 12345, $$postgres-secret$$, q'[oracle-secret]'
|
||||
FROM users /* block-secret */
|
||||
WHERE password = bare-secret
|
||||
AND endpoint = 'postgres://alice:db-secret@example.test/app'
|
||||
AND note = "quoted-secret"`
|
||||
|
||||
redacted := RedactSQL(input)
|
||||
for _, secret := range []string{
|
||||
"comment-secret", "alice-secret", "12345", "postgres-secret", "oracle-secret",
|
||||
"block-secret", "bare-secret", "alice", "db-secret", "quoted-secret",
|
||||
} {
|
||||
if strings.Contains(redacted, secret) {
|
||||
t.Fatalf("redacted SQL leaked %q: %s", secret, redacted)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(redacted, "SELECT") || !strings.Contains(redacted, "FROM users") {
|
||||
t.Fatalf("redaction should retain SQL structure: %s", redacted)
|
||||
}
|
||||
if !strings.Contains(redacted, "password = ?") {
|
||||
t.Fatalf("sensitive unquoted assignment should be replaced: %s", redacted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedactSQLPreservesIdentifiersAndPlaceholders(t *testing.T) {
|
||||
input := "SELECT report_2026, `OrderID`, [AccountID] FROM events WHERE id = $1 AND name = ?"
|
||||
redacted := RedactSQL(input)
|
||||
for _, fragment := range []string{"report_2026", "`OrderID`", "[AccountID]", "$1", "name = ?"} {
|
||||
if !strings.Contains(redacted, fragment) {
|
||||
t.Fatalf("expected %q to remain in %q", fragment, redacted)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedactErrorRemovesSecretsAndControlCharacters(t *testing.T) {
|
||||
input := "connect postgres://alice:db-secret@example.test/app password=top-secret " +
|
||||
"Authorization: Bearer abc.def.ghi Basic dXNlcjpwYXNz fallback alice:bare-pass@db.internal oracle scott/tiger@dbhost/service near 'literal-secret'\nnext"
|
||||
redacted := RedactError(input)
|
||||
for _, secret := range []string{"alice", "db-secret", "top-secret", "abc.def.ghi", "dXNlcjpwYXNz", "bare-pass", "scott", "tiger", "service", "literal-secret"} {
|
||||
if strings.Contains(redacted, secret) {
|
||||
t.Fatalf("redacted error leaked %q: %s", secret, redacted)
|
||||
}
|
||||
}
|
||||
if strings.ContainsAny(redacted, "\r\n") {
|
||||
t.Fatalf("redacted error should be single line: %q", redacted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedactSQLRemovesNestedBlockComments(t *testing.T) {
|
||||
redacted := RedactSQL("SELECT 1 /* outer /* nested */ private-after-nested */ FROM users")
|
||||
if strings.Contains(redacted, "outer") || strings.Contains(redacted, "nested") || strings.Contains(redacted, "private-after-nested") {
|
||||
t.Fatalf("nested SQL comment leaked into audit text: %q", redacted)
|
||||
}
|
||||
if !strings.Contains(redacted, "FROM users") {
|
||||
t.Fatalf("nested comment redaction removed following SQL structure: %q", redacted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedactSQLPreservesSensitiveNamedColumnsWithoutAssignments(t *testing.T) {
|
||||
input := "SELECT password, token, secret FROM sessions"
|
||||
if redacted := RedactSQL(input); redacted != input {
|
||||
t.Fatalf("sensitive-named columns were mistaken for assigned values: %q", redacted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedactErrorRemovesUnquotedDuplicateKeyValues(t *testing.T) {
|
||||
input := "ERROR: duplicate key value violates unique constraint; DETAIL: Key (email, tenant_id)=(alice@example.test, 42) already exists; The duplicate key value is (private-token). Failing row contains (7, failing@example.test, row-secret)."
|
||||
redacted := RedactError(input)
|
||||
for _, secret := range []string{"alice@example.test", "42", "private-token", "failing@example.test", "row-secret"} {
|
||||
if strings.Contains(redacted, secret) {
|
||||
t.Fatalf("redacted duplicate-key error leaked %q: %s", secret, redacted)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(redacted, "Key (email, tenant_id)=(?)") {
|
||||
t.Fatalf("redacted error lost safe constraint structure: %s", redacted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedactionBoundsStoredText(t *testing.T) {
|
||||
longSQL := "SELECT '" + strings.Repeat("界", maxSQLRunes+100) + "'"
|
||||
if got := len([]rune(RedactSQL(longSQL))); got > maxSQLRunes {
|
||||
t.Fatalf("redacted SQL exceeds limit: %d", got)
|
||||
}
|
||||
longError := strings.Repeat("failure ", maxErrorRunes)
|
||||
if got := len([]rune(RedactError(longError))); got > maxErrorRunes {
|
||||
t.Fatalf("redacted error exceeds limit: %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedactQueryUsesSQLLexerForSQLTypes(t *testing.T) {
|
||||
query := "SELECT * FROM users WHERE password = 'sql-secret' AND id = 42"
|
||||
for _, dbType := range []string{"mysql", "postgres", "sqlserver", "oracle", "clickhouse", "custom"} {
|
||||
if got, want := RedactQuery(dbType, query), RedactSQL(query); got != want {
|
||||
t.Fatalf("RedactQuery(%s)=%q, want SQL redaction %q", dbType, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedactQueryRedactsRedisValuesAndPreservesCommandsAndKeys(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
query string
|
||||
want string
|
||||
secrets []string
|
||||
}{
|
||||
{name: "auth", query: "AUTH default auth-secret", want: "AUTH ? ?", secrets: []string{"default", "auth-secret"}},
|
||||
{name: "hello auth", query: "HELLO 3 AUTH app hello-secret SETNAME client-secret", want: "HELLO 3 AUTH ? ? SETNAME ?", secrets: []string{"app", "hello-secret", "client-secret"}},
|
||||
{name: "config set", query: "CONFIG SET requirepass config-secret", want: "CONFIG SET requirepass ?", secrets: []string{"config-secret"}},
|
||||
{name: "set", query: "SET session:key set-secret EX 60", want: "SET session:key ?", secrets: []string{"set-secret", "60"}},
|
||||
{name: "mset", query: "MSET key:1 first-secret key:2 second-secret", want: "MSET key:1 ? key:2 ?", secrets: []string{"first-secret", "second-secret"}},
|
||||
{name: "hset", query: "HSET profile:1 email private@example.test token hash-secret", want: "HSET profile:1 email ? token ?", secrets: []string{"private@example.test", "hash-secret"}},
|
||||
{name: "list", query: "LPUSH queue:key payload-one payload-two", want: "LPUSH queue:key ?", secrets: []string{"payload-one", "payload-two"}},
|
||||
{name: "set member", query: "SADD set:key member-secret", want: "SADD set:key ?", secrets: []string{"member-secret"}},
|
||||
{name: "sorted set", query: "ZADD rank:key 12.5 member-secret", want: "ZADD rank:key ?", secrets: []string{"12.5", "member-secret"}},
|
||||
{name: "publish", query: "PUBLISH channel:key message-secret", want: "PUBLISH channel:key ?", secrets: []string{"message-secret"}},
|
||||
{name: "stream", query: "XADD stream:key * field payload-secret", want: "XADD stream:key ?", secrets: []string{"payload-secret", "field"}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := RedactQuery("redis", tt.query)
|
||||
if got != tt.want {
|
||||
t.Fatalf("RedactQuery(redis)=%q, want %q", got, tt.want)
|
||||
}
|
||||
for _, secret := range tt.secrets {
|
||||
if strings.Contains(got, secret) {
|
||||
t.Fatalf("Redis redaction leaked %q in %q", secret, got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedactQueryDefaultsMessageAndUnknownLanguagesToMetadataOnly(t *testing.T) {
|
||||
query := `PUBLISH topic {"token":"message-secret"}`
|
||||
for _, dbType := range []string{"rocketmq", "mqtt", "kafka", "rabbitmq", "mongodb", "elasticsearch", "milvus", "mystery-driver"} {
|
||||
if got := RedactQuery(dbType, query); got != "" {
|
||||
t.Fatalf("RedactQuery(%s)=%q, want metadata-only empty text", dbType, got)
|
||||
}
|
||||
}
|
||||
if got := RedactQuery("redis", `SET key "unterminated-secret`); got != "" {
|
||||
t.Fatalf("unparseable Redis input must fall back to metadata-only, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizeEventPreservesPendingCommitMode(t *testing.T) {
|
||||
event := sanitizeEvent(Event{
|
||||
EventType: "transaction_begin",
|
||||
Status: "success",
|
||||
CommitMode: CommitModePending,
|
||||
}, Settings{
|
||||
Enabled: true,
|
||||
CaptureMode: CaptureModeRedacted,
|
||||
RetentionDays: 30,
|
||||
MaxRecords: 100,
|
||||
})
|
||||
if event.CommitMode != CommitModePending {
|
||||
t.Fatalf("pending commit mode = %q, want %q", event.CommitMode, CommitModePending)
|
||||
}
|
||||
}
|
||||
880
internal/sqlaudit/store.go
Normal file
880
internal/sqlaudit/store.go
Normal file
@@ -0,0 +1,880 @@
|
||||
package sqlaudit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
const (
|
||||
schemaVersion = 1
|
||||
chainVersion = "sqlaudit-chain-v1"
|
||||
integrityAlgorithm = "sha256-canonical-v1"
|
||||
maxRetentionDays = 3650
|
||||
maxConfiguredEvents = 10_000_000
|
||||
)
|
||||
|
||||
var (
|
||||
ErrClosed = errors.New("sql audit store is closed")
|
||||
ErrBatchExceedsMaxRecords = errors.New("sql audit batch exceeds configured max records")
|
||||
)
|
||||
|
||||
type Store struct {
|
||||
db *sql.DB
|
||||
path string
|
||||
}
|
||||
|
||||
func Open(path string) (*Store, error) {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
return nil, errors.New("sql audit database path is empty")
|
||||
}
|
||||
absPath, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve sql audit database path: %w", err)
|
||||
}
|
||||
dir := filepath.Dir(absPath)
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return nil, fmt.Errorf("create sql audit directory: %w", err)
|
||||
}
|
||||
if err := os.Chmod(dir, 0o700); err != nil {
|
||||
return nil, fmt.Errorf("secure sql audit directory: %w", err)
|
||||
}
|
||||
|
||||
database, err := sql.Open("sqlite", sqliteAuditDSN(absPath))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open sql audit database: %w", err)
|
||||
}
|
||||
database.SetMaxOpenConns(4)
|
||||
database.SetMaxIdleConns(4)
|
||||
store := &Store{db: database, path: absPath}
|
||||
if err := store.initialize(); err != nil {
|
||||
_ = database.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := os.Chmod(absPath, 0o600); err != nil {
|
||||
_ = database.Close()
|
||||
return nil, fmt.Errorf("secure sql audit database: %w", err)
|
||||
}
|
||||
return store, nil
|
||||
}
|
||||
|
||||
func (s *Store) Path() string {
|
||||
if s == nil {
|
||||
return ""
|
||||
}
|
||||
return s.path
|
||||
}
|
||||
|
||||
func (s *Store) Close() error {
|
||||
if s == nil || s.db == nil {
|
||||
return nil
|
||||
}
|
||||
// Consolidate WAL content before a data-root migration or application exit.
|
||||
// App lifecycle locking ensures no new store operation starts during Close.
|
||||
_, checkpointErr := s.db.ExecContext(context.Background(), `PRAGMA wal_checkpoint(TRUNCATE)`)
|
||||
err := errors.Join(checkpointErr, s.db.Close())
|
||||
s.db = nil
|
||||
return err
|
||||
}
|
||||
|
||||
func sqliteAuditDSN(path string) string {
|
||||
uriPath := filepath.ToSlash(path)
|
||||
dsn := &url.URL{Scheme: "file", Path: uriPath}
|
||||
if runtime.GOOS == "windows" {
|
||||
if strings.HasPrefix(uriPath, "//") {
|
||||
withoutPrefix := strings.TrimPrefix(uriPath, "//")
|
||||
if separator := strings.IndexByte(withoutPrefix, '/'); separator >= 0 {
|
||||
dsn.Host = withoutPrefix[:separator]
|
||||
dsn.Path = withoutPrefix[separator:]
|
||||
}
|
||||
} else if !strings.HasPrefix(uriPath, "/") {
|
||||
dsn.Path = "/" + uriPath
|
||||
}
|
||||
}
|
||||
query := url.Values{}
|
||||
for _, pragma := range []string{
|
||||
"busy_timeout(5000)",
|
||||
"foreign_keys(ON)",
|
||||
"synchronous(FULL)",
|
||||
"journal_mode(WAL)",
|
||||
} {
|
||||
query.Add("_pragma", pragma)
|
||||
}
|
||||
dsn.RawQuery = query.Encode()
|
||||
return dsn.String()
|
||||
}
|
||||
|
||||
func (s *Store) initialize() error {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx := context.Background()
|
||||
for _, pragma := range []string{
|
||||
"PRAGMA busy_timeout=5000",
|
||||
"PRAGMA journal_mode=WAL",
|
||||
"PRAGMA synchronous=FULL",
|
||||
"PRAGMA foreign_keys=ON",
|
||||
} {
|
||||
if _, err := s.db.ExecContext(ctx, pragma); err != nil {
|
||||
return fmt.Errorf("configure sql audit database (%s): %w", pragma, err)
|
||||
}
|
||||
}
|
||||
var existingSchemaVersion int
|
||||
if err := s.db.QueryRowContext(ctx, `PRAGMA user_version`).Scan(&existingSchemaVersion); err != nil {
|
||||
return fmt.Errorf("read SQL audit schema version: %w", err)
|
||||
}
|
||||
if existingSchemaVersion > schemaVersion {
|
||||
return fmt.Errorf("SQL audit schema version %d is newer than supported version %d", existingSchemaVersion, schemaVersion)
|
||||
}
|
||||
|
||||
statements := []string{
|
||||
`CREATE TABLE IF NOT EXISTS sql_audit_events (
|
||||
sequence INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
id TEXT NOT NULL UNIQUE,
|
||||
timestamp INTEGER NOT NULL,
|
||||
event_type TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
connection_id TEXT NOT NULL DEFAULT '',
|
||||
connection_fingerprint TEXT NOT NULL,
|
||||
db_type TEXT NOT NULL DEFAULT '',
|
||||
database_name TEXT NOT NULL DEFAULT '',
|
||||
query_id TEXT NOT NULL DEFAULT '',
|
||||
transaction_id TEXT NOT NULL DEFAULT '',
|
||||
source TEXT NOT NULL DEFAULT '',
|
||||
boundary_mode TEXT NOT NULL DEFAULT 'unknown',
|
||||
commit_mode TEXT NOT NULL DEFAULT '',
|
||||
sql_text TEXT NOT NULL DEFAULT '',
|
||||
sql_redacted INTEGER NOT NULL DEFAULT 1,
|
||||
sql_fingerprint TEXT NOT NULL,
|
||||
statement_index INTEGER NOT NULL DEFAULT 0,
|
||||
statement_count INTEGER NOT NULL DEFAULT 0,
|
||||
duration_ms INTEGER NOT NULL DEFAULT 0,
|
||||
rows_affected INTEGER NOT NULL DEFAULT 0,
|
||||
rows_returned INTEGER NOT NULL DEFAULT 0,
|
||||
error_text TEXT NOT NULL DEFAULT '',
|
||||
prev_hash TEXT NOT NULL DEFAULT '',
|
||||
record_hash TEXT NOT NULL
|
||||
)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_sql_audit_events_timestamp ON sql_audit_events(timestamp DESC, sequence DESC)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_sql_audit_events_connection ON sql_audit_events(connection_id, timestamp DESC)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_sql_audit_events_transaction ON sql_audit_events(transaction_id, sequence)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_sql_audit_events_type_status ON sql_audit_events(event_type, status, timestamp DESC)`,
|
||||
`CREATE TABLE IF NOT EXISTS sql_audit_settings (
|
||||
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
|
||||
enabled INTEGER NOT NULL,
|
||||
capture_mode TEXT NOT NULL,
|
||||
retention_days INTEGER NOT NULL,
|
||||
max_records INTEGER NOT NULL
|
||||
)`,
|
||||
`INSERT OR IGNORE INTO sql_audit_settings(singleton, enabled, capture_mode, retention_days, max_records)
|
||||
VALUES(1, 1, 'redacted', 30, 100000)`,
|
||||
fmt.Sprintf("PRAGMA user_version=%d", schemaVersion),
|
||||
}
|
||||
for _, statement := range statements {
|
||||
if _, err := s.db.ExecContext(ctx, statement); err != nil {
|
||||
return fmt.Errorf("initialize sql audit database: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Append adds one terminal audit fact. When auditing is disabled this method is
|
||||
// a successful no-op. SQL and error text are sanitized before the transaction.
|
||||
func (s *Store) Append(event Event) error {
|
||||
return s.AppendBatch([]Event{event})
|
||||
}
|
||||
|
||||
// AppendControl persists an audit-control event even when ordinary collection
|
||||
// is disabled. It is reserved for settings and purge boundaries so disabling
|
||||
// or clearing the audit stream cannot create an unmarked control-plane window.
|
||||
func (s *Store) AppendControl(event Event) error {
|
||||
return s.appendBatch([]Event{event}, true)
|
||||
}
|
||||
|
||||
// AppendBatch persists related facts in one durable SQLite transaction. It is
|
||||
// primarily used for the statements executed inside one managed transaction,
|
||||
// avoiding one FULL-sync round trip per statement while keeping the complete
|
||||
// batch visible before the database transaction is returned to the caller.
|
||||
func (s *Store) AppendBatch(events []Event) error {
|
||||
return s.appendBatch(events, false)
|
||||
}
|
||||
|
||||
func (s *Store) appendBatch(events []Event, bypassDisabled bool) error {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(events) == 0 {
|
||||
return nil
|
||||
}
|
||||
inputEvents := events
|
||||
|
||||
return s.withImmediate(func(conn *sql.Conn) error {
|
||||
settings, err := getSettingsFrom(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !settings.Enabled && !bypassDisabled {
|
||||
return nil
|
||||
}
|
||||
return appendEventsLocked(conn, settings, inputEvents)
|
||||
})
|
||||
}
|
||||
|
||||
func appendEventsLocked(conn *sql.Conn, settings Settings, inputEvents []Event) error {
|
||||
if len(inputEvents) > settings.MaxRecords {
|
||||
return fmt.Errorf(
|
||||
"%w: batch=%d limit=%d",
|
||||
ErrBatchExceedsMaxRecords,
|
||||
len(inputEvents),
|
||||
settings.MaxRecords,
|
||||
)
|
||||
}
|
||||
prepared := append([]Event(nil), inputEvents...)
|
||||
now := time.Now().UnixMilli()
|
||||
for index := range prepared {
|
||||
event := &prepared[index]
|
||||
if event.Timestamp <= 0 {
|
||||
event.Timestamp = now
|
||||
}
|
||||
if strings.TrimSpace(event.ID) == "" {
|
||||
event.ID = uuid.NewString()
|
||||
} else {
|
||||
event.ID = sanitizeLabel(event.ID, 256)
|
||||
}
|
||||
if strings.TrimSpace(event.EventType) == "" {
|
||||
return fmt.Errorf("sql audit event %d type is required", index+1)
|
||||
}
|
||||
if strings.TrimSpace(event.Status) == "" {
|
||||
return fmt.Errorf("sql audit event %d status is required", index+1)
|
||||
}
|
||||
}
|
||||
if _, err := pruneLocked(conn, settings, time.Now(), int64(len(prepared))); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var previousHash string
|
||||
err := conn.QueryRowContext(context.Background(),
|
||||
`SELECT record_hash FROM sql_audit_events ORDER BY sequence DESC LIMIT 1`,
|
||||
).Scan(&previousHash)
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
return fmt.Errorf("read sql audit chain head: %w", err)
|
||||
}
|
||||
for index := range prepared {
|
||||
event := sanitizeEvent(prepared[index], settings)
|
||||
event.PrevHash = previousHash
|
||||
|
||||
result, err := conn.ExecContext(context.Background(), `INSERT INTO sql_audit_events(
|
||||
id, timestamp, event_type, status, connection_id, connection_fingerprint,
|
||||
db_type, database_name, query_id, transaction_id, source, boundary_mode,
|
||||
commit_mode, sql_text, sql_redacted, sql_fingerprint, statement_index,
|
||||
statement_count, duration_ms, rows_affected, rows_returned, error_text,
|
||||
prev_hash, record_hash
|
||||
) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
||||
event.ID, event.Timestamp, event.EventType, event.Status, event.ConnectionID,
|
||||
event.ConnectionFingerprint, event.DBType, event.Database, event.QueryID,
|
||||
event.TransactionID, event.Source, event.BoundaryMode, event.CommitMode,
|
||||
event.SQLText, boolToInt(event.SQLRedacted), event.SQLFingerprint,
|
||||
event.StatementIndex, event.StatementCount, event.DurationMs,
|
||||
event.RowsAffected, event.RowsReturned, event.Error, event.PrevHash, "",
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("append sql audit event %d: %w", index+1, err)
|
||||
}
|
||||
event.Sequence, err = result.LastInsertId()
|
||||
if err != nil {
|
||||
return fmt.Errorf("read sql audit sequence for event %d: %w", index+1, err)
|
||||
}
|
||||
event.Hash, err = calculateEventHash(event)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := conn.ExecContext(context.Background(),
|
||||
`UPDATE sql_audit_events SET record_hash=? WHERE sequence=?`, event.Hash, event.Sequence,
|
||||
); err != nil {
|
||||
return fmt.Errorf("finalize sql audit event %d hash: %w", index+1, err)
|
||||
}
|
||||
previousHash = event.Hash
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) Query(filter Filter) (Page, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return Page{}, err
|
||||
}
|
||||
filter = normalizeFilter(filter)
|
||||
where, args := buildFilterWhere(filter)
|
||||
page := Page{Items: []Event{}, Page: filter.Page, PageSize: filter.PageSize}
|
||||
|
||||
summarySQL := `SELECT COUNT(*),
|
||||
COALESCE(SUM(CASE WHEN status='success' THEN 1 ELSE 0 END), 0),
|
||||
COALESCE(SUM(CASE WHEN status='error' THEN 1 ELSE 0 END), 0),
|
||||
COUNT(DISTINCT CASE WHEN transaction_id <> '' THEN transaction_id END)
|
||||
FROM sql_audit_events` + where
|
||||
if err := s.db.QueryRowContext(context.Background(), summarySQL, args...).Scan(
|
||||
&page.Summary.TotalEvents,
|
||||
&page.Summary.SuccessCount,
|
||||
&page.Summary.ErrorCount,
|
||||
&page.Summary.TransactionCount,
|
||||
); err != nil {
|
||||
return Page{}, fmt.Errorf("summarize sql audit events: %w", err)
|
||||
}
|
||||
page.Total = page.Summary.TotalEvents
|
||||
|
||||
queryArgs := append(append([]any{}, args...), filter.PageSize, (filter.Page-1)*filter.PageSize)
|
||||
rows, err := s.db.QueryContext(context.Background(), selectEventColumns+where+
|
||||
` ORDER BY timestamp DESC, sequence DESC LIMIT ? OFFSET ?`, queryArgs...)
|
||||
if err != nil {
|
||||
return Page{}, fmt.Errorf("query sql audit events: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
event, err := scanEvent(rows)
|
||||
if err != nil {
|
||||
return Page{}, err
|
||||
}
|
||||
page.Items = append(page.Items, event)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return Page{}, fmt.Errorf("iterate sql audit events: %w", err)
|
||||
}
|
||||
return page, nil
|
||||
}
|
||||
|
||||
func (s *Store) UpdateSettings(settings Settings) error {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return err
|
||||
}
|
||||
normalized, err := normalizeSettings(settings)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.withImmediate(func(conn *sql.Conn) error {
|
||||
if _, err := conn.ExecContext(context.Background(), `UPDATE sql_audit_settings
|
||||
SET enabled=?, capture_mode=?, retention_days=?, max_records=? WHERE singleton=1`,
|
||||
boolToInt(normalized.Enabled), normalized.CaptureMode,
|
||||
normalized.RetentionDays, normalized.MaxRecords,
|
||||
); err != nil {
|
||||
return fmt.Errorf("update sql audit settings: %w", err)
|
||||
}
|
||||
_, err := pruneLocked(conn, normalized, time.Now(), 0)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
// UpdateSettingsWithControl atomically applies settings and appends a
|
||||
// non-disableable control boundary describing the old and new safe values.
|
||||
func (s *Store) UpdateSettingsWithControl(settings Settings, control Event) error {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return err
|
||||
}
|
||||
normalized, err := normalizeSettings(settings)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.withImmediate(func(conn *sql.Conn) error {
|
||||
previous, err := getSettingsFrom(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := conn.ExecContext(context.Background(), `UPDATE sql_audit_settings
|
||||
SET enabled=?, capture_mode=?, retention_days=?, max_records=? WHERE singleton=1`,
|
||||
boolToInt(normalized.Enabled), normalized.CaptureMode,
|
||||
normalized.RetentionDays, normalized.MaxRecords,
|
||||
); err != nil {
|
||||
return fmt.Errorf("update sql audit settings: %w", err)
|
||||
}
|
||||
control.DBType = "gonavi"
|
||||
control.SQLText = settingsControlDescriptor(previous, normalized)
|
||||
return appendEventsLocked(conn, controlAuditSettings(normalized), []Event{control})
|
||||
})
|
||||
}
|
||||
|
||||
// Control descriptors contain only GoNavi-owned structural metadata. Keep
|
||||
// them visible even when ordinary events use metadata-only capture, otherwise
|
||||
// disable/retention/purge boundaries would lose the values needed to explain
|
||||
// an audit window.
|
||||
func controlAuditSettings(settings Settings) Settings {
|
||||
settings.CaptureMode = CaptureModeRedacted
|
||||
return settings
|
||||
}
|
||||
|
||||
func settingsControlDescriptor(previous, next Settings) string {
|
||||
return fmt.Sprintf(
|
||||
"AUDIT SETTINGS FROM_ENABLED_%s TO_ENABLED_%s FROM_CAPTURE_%s TO_CAPTURE_%s FROM_RETENTION_DAYS_%d TO_RETENTION_DAYS_%d FROM_MAX_RECORDS_%d TO_MAX_RECORDS_%d",
|
||||
settingsBoolToken(previous.Enabled),
|
||||
settingsBoolToken(next.Enabled),
|
||||
strings.ToUpper(previous.CaptureMode),
|
||||
strings.ToUpper(next.CaptureMode),
|
||||
previous.RetentionDays,
|
||||
next.RetentionDays,
|
||||
previous.MaxRecords,
|
||||
next.MaxRecords,
|
||||
)
|
||||
}
|
||||
|
||||
func settingsBoolToken(value bool) string {
|
||||
if value {
|
||||
return "ON"
|
||||
}
|
||||
return "OFF"
|
||||
}
|
||||
|
||||
func (s *Store) GetSettings() (Settings, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return Settings{}, err
|
||||
}
|
||||
return getSettingsFrom(s.db)
|
||||
}
|
||||
|
||||
// Clear deletes the oldest contiguous sequence prefix whose timestamps are
|
||||
// older than beforeTimestamp. A non-positive timestamp clears all events.
|
||||
// Remaining records and their hashes are never rewritten; integrity checks
|
||||
// report a truncated/partial chain when a retained first record still points
|
||||
// at a deleted predecessor.
|
||||
func (s *Store) Clear(beforeTimestamp int64) (int64, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var deleted int64
|
||||
err := s.withImmediate(func(conn *sql.Conn) error {
|
||||
var err error
|
||||
if beforeTimestamp <= 0 {
|
||||
deleted, err = deleteAllLocked(conn)
|
||||
} else {
|
||||
deleted, err = deleteTimestampPrefixLocked(conn, beforeTimestamp)
|
||||
}
|
||||
return err
|
||||
})
|
||||
return deleted, err
|
||||
}
|
||||
|
||||
// ClearWithControl atomically removes the selected prefix and appends a control
|
||||
// boundary that remains visible even when ordinary collection is disabled.
|
||||
func (s *Store) ClearWithControl(beforeTimestamp int64, control Event) (int64, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var deleted int64
|
||||
err := s.withImmediate(func(conn *sql.Conn) error {
|
||||
settings, err := getSettingsFrom(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if beforeTimestamp <= 0 {
|
||||
deleted, err = deleteAllLocked(conn)
|
||||
control.SQLText = "AUDIT CLEAR ALL"
|
||||
} else {
|
||||
deleted, err = deleteTimestampPrefixLocked(conn, beforeTimestamp)
|
||||
control.SQLText = fmt.Sprintf("AUDIT CLEAR BEFORE_TIMESTAMP_%d", beforeTimestamp)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
control.DBType = "gonavi"
|
||||
control.RowsAffected = deleted
|
||||
return appendEventsLocked(conn, controlAuditSettings(settings), []Event{control})
|
||||
})
|
||||
return deleted, err
|
||||
}
|
||||
|
||||
func (s *Store) VerifyIntegrity() (IntegrityReport, error) {
|
||||
report := IntegrityReport{
|
||||
Valid: true,
|
||||
WeakValidation: true,
|
||||
Algorithm: integrityAlgorithm,
|
||||
Message: "ok (unkeyed local hash chain; weak validation)",
|
||||
}
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return report, err
|
||||
}
|
||||
rows, err := s.db.QueryContext(context.Background(), selectEventColumns+` ORDER BY sequence ASC`)
|
||||
if err != nil {
|
||||
return report, fmt.Errorf("read sql audit chain: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
previousHash := ""
|
||||
var previousSequence int64
|
||||
firstRecord := true
|
||||
for rows.Next() {
|
||||
event, err := scanEvent(rows)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
report.CheckedRecords++
|
||||
if firstRecord {
|
||||
report.FirstSequence = event.Sequence
|
||||
}
|
||||
report.LastSequence = event.Sequence
|
||||
if event.Sequence <= 0 {
|
||||
return invalidIntegrityReport(report, event.Sequence, "sequence must be positive"), nil
|
||||
}
|
||||
isFirst := firstRecord
|
||||
if isFirst {
|
||||
report.PartialChain = event.Sequence > 1 || event.PrevHash != ""
|
||||
report.TruncatedPrefix = report.PartialChain
|
||||
if report.PartialChain {
|
||||
report.Message = "ok (partial chain after prefix retention; unkeyed weak validation)"
|
||||
}
|
||||
} else if event.Sequence <= previousSequence {
|
||||
return invalidIntegrityReport(report, event.Sequence, "sequence order is invalid"), nil
|
||||
}
|
||||
if !isFirst && event.PrevHash != previousHash {
|
||||
return invalidIntegrityReport(report, event.Sequence, "previous hash does not match"), nil
|
||||
}
|
||||
expected, err := calculateEventHash(event)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if event.Hash != expected {
|
||||
return invalidIntegrityReport(report, event.Sequence, "record hash does not match"), nil
|
||||
}
|
||||
previousSequence = event.Sequence
|
||||
previousHash = event.Hash
|
||||
firstRecord = false
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return report, fmt.Errorf("iterate sql audit chain: %w", err)
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func (s *Store) ensureOpen() error {
|
||||
if s == nil || s.db == nil {
|
||||
return ErrClosed
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) withImmediate(operation func(*sql.Conn) error) (resultErr error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx := context.Background()
|
||||
conn, err := s.db.Conn(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("acquire sql audit connection: %w", err)
|
||||
}
|
||||
defer func() { resultErr = errors.Join(resultErr, conn.Close()) }()
|
||||
if _, err := conn.ExecContext(ctx, "BEGIN IMMEDIATE"); err != nil {
|
||||
return fmt.Errorf("begin sql audit transaction: %w", err)
|
||||
}
|
||||
committed := false
|
||||
defer func() {
|
||||
if !committed {
|
||||
_, rollbackErr := conn.ExecContext(context.Background(), "ROLLBACK")
|
||||
resultErr = errors.Join(resultErr, rollbackErr)
|
||||
}
|
||||
}()
|
||||
if operation != nil {
|
||||
if err := operation(conn); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err := conn.ExecContext(ctx, "COMMIT"); err != nil {
|
||||
return fmt.Errorf("commit sql audit transaction: %w", err)
|
||||
}
|
||||
committed = true
|
||||
return nil
|
||||
}
|
||||
|
||||
type queryRower interface {
|
||||
QueryRowContext(context.Context, string, ...any) *sql.Row
|
||||
}
|
||||
|
||||
func getSettingsFrom(queryer queryRower) (Settings, error) {
|
||||
var enabled int
|
||||
var settings Settings
|
||||
err := queryer.QueryRowContext(context.Background(), `SELECT enabled, capture_mode, retention_days, max_records
|
||||
FROM sql_audit_settings WHERE singleton=1`).Scan(
|
||||
&enabled, &settings.CaptureMode, &settings.RetentionDays, &settings.MaxRecords,
|
||||
)
|
||||
if err != nil {
|
||||
return Settings{}, fmt.Errorf("read sql audit settings: %w", err)
|
||||
}
|
||||
settings.Enabled = enabled != 0
|
||||
normalized, err := normalizeSettings(settings)
|
||||
if err != nil {
|
||||
return Settings{}, fmt.Errorf("invalid persisted sql audit settings: %w", err)
|
||||
}
|
||||
return normalized, nil
|
||||
}
|
||||
|
||||
func normalizeSettings(settings Settings) (Settings, error) {
|
||||
settings.CaptureMode = strings.ToLower(strings.TrimSpace(settings.CaptureMode))
|
||||
if settings.CaptureMode == "" {
|
||||
settings.CaptureMode = CaptureModeRedacted
|
||||
}
|
||||
if settings.CaptureMode != CaptureModeRedacted && settings.CaptureMode != CaptureModeMetadata {
|
||||
return Settings{}, fmt.Errorf("unsupported SQL audit capture mode %q", settings.CaptureMode)
|
||||
}
|
||||
if settings.RetentionDays <= 0 {
|
||||
settings.RetentionDays = defaultRetentionDays
|
||||
}
|
||||
if settings.RetentionDays > maxRetentionDays {
|
||||
return Settings{}, fmt.Errorf("SQL audit retention days cannot exceed %d", maxRetentionDays)
|
||||
}
|
||||
if settings.MaxRecords <= 0 {
|
||||
settings.MaxRecords = defaultMaxRecords
|
||||
}
|
||||
if settings.MaxRecords > maxConfiguredEvents {
|
||||
return Settings{}, fmt.Errorf("SQL audit max records cannot exceed %d", maxConfiguredEvents)
|
||||
}
|
||||
return settings, nil
|
||||
}
|
||||
|
||||
func normalizeFilter(filter Filter) Filter {
|
||||
filter.Search = truncateRunes(strings.TrimSpace(filter.Search), 512)
|
||||
filter.ConnectionID = strings.TrimSpace(filter.ConnectionID)
|
||||
filter.Database = strings.TrimSpace(filter.Database)
|
||||
filter.DBType = strings.ToLower(strings.TrimSpace(filter.DBType))
|
||||
filter.EventType = strings.ToLower(strings.TrimSpace(filter.EventType))
|
||||
filter.Status = strings.ToLower(strings.TrimSpace(filter.Status))
|
||||
filter.TransactionID = strings.TrimSpace(filter.TransactionID)
|
||||
filter.Source = strings.ToLower(strings.TrimSpace(filter.Source))
|
||||
if filter.Page <= 0 {
|
||||
filter.Page = 1
|
||||
}
|
||||
if filter.PageSize <= 0 {
|
||||
filter.PageSize = defaultPageSize
|
||||
}
|
||||
if filter.PageSize > maxPageSize {
|
||||
filter.PageSize = maxPageSize
|
||||
}
|
||||
return filter
|
||||
}
|
||||
|
||||
func buildFilterWhere(filter Filter) (string, []any) {
|
||||
conditions := make([]string, 0, 10)
|
||||
args := make([]any, 0, 12)
|
||||
if filter.Search != "" {
|
||||
pattern := "%" + escapeLike(filter.Search) + "%"
|
||||
conditions = append(conditions, `(sql_text LIKE ? ESCAPE '\' OR error_text LIKE ? ESCAPE '\'
|
||||
OR connection_id LIKE ? ESCAPE '\' OR connection_fingerprint LIKE ? ESCAPE '\'
|
||||
OR database_name LIKE ? ESCAPE '\' OR db_type LIKE ? ESCAPE '\'
|
||||
OR query_id LIKE ? ESCAPE '\' OR transaction_id LIKE ? ESCAPE '\'
|
||||
OR sql_fingerprint LIKE ? ESCAPE '\' OR source LIKE ? ESCAPE '\')`)
|
||||
for range 10 {
|
||||
args = append(args, pattern)
|
||||
}
|
||||
}
|
||||
appendExact := func(column string, value string) {
|
||||
if value == "" {
|
||||
return
|
||||
}
|
||||
conditions = append(conditions, column+" = ?")
|
||||
args = append(args, value)
|
||||
}
|
||||
appendExact("connection_id", filter.ConnectionID)
|
||||
appendExact("database_name", filter.Database)
|
||||
appendExact("db_type", filter.DBType)
|
||||
appendExact("event_type", filter.EventType)
|
||||
appendExact("status", filter.Status)
|
||||
appendExact("transaction_id", filter.TransactionID)
|
||||
appendExact("source", filter.Source)
|
||||
if filter.FromTimestamp > 0 {
|
||||
conditions = append(conditions, "timestamp >= ?")
|
||||
args = append(args, filter.FromTimestamp)
|
||||
}
|
||||
if filter.ToTimestamp > 0 {
|
||||
conditions = append(conditions, "timestamp <= ?")
|
||||
args = append(args, filter.ToTimestamp)
|
||||
}
|
||||
if len(conditions) == 0 {
|
||||
return "", args
|
||||
}
|
||||
return " WHERE " + strings.Join(conditions, " AND "), args
|
||||
}
|
||||
|
||||
func escapeLike(value string) string {
|
||||
value = strings.ReplaceAll(value, `\`, `\\`)
|
||||
value = strings.ReplaceAll(value, `%`, `\%`)
|
||||
return strings.ReplaceAll(value, `_`, `\_`)
|
||||
}
|
||||
|
||||
func pruneLocked(conn *sql.Conn, settings Settings, now time.Time, reserve int64) (int64, error) {
|
||||
var deleted int64
|
||||
cutoff := now.Add(-time.Duration(settings.RetentionDays) * 24 * time.Hour).UnixMilli()
|
||||
expired, err := deleteTimestampPrefixLocked(conn, cutoff)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("prune expired sql audit events: %w", err)
|
||||
}
|
||||
deleted += expired
|
||||
|
||||
var current int64
|
||||
if err := conn.QueryRowContext(context.Background(), `SELECT COUNT(*) FROM sql_audit_events`).Scan(¤t); err != nil {
|
||||
return 0, fmt.Errorf("count sql audit events: %w", err)
|
||||
}
|
||||
target := int64(settings.MaxRecords) - reserve
|
||||
if target < 0 {
|
||||
target = 0
|
||||
}
|
||||
if current > target {
|
||||
removeCount := current - target
|
||||
result, err := conn.ExecContext(context.Background(), `DELETE FROM sql_audit_events WHERE sequence IN (
|
||||
SELECT sequence FROM sql_audit_events ORDER BY sequence ASC LIMIT ?
|
||||
)`, removeCount)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("enforce sql audit record limit: %w", err)
|
||||
}
|
||||
if count, err := result.RowsAffected(); err == nil {
|
||||
deleted += count
|
||||
}
|
||||
}
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
func deleteAllLocked(conn *sql.Conn) (int64, error) {
|
||||
result, err := conn.ExecContext(context.Background(), `DELETE FROM sql_audit_events`)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("clear SQL audit events: %w", err)
|
||||
}
|
||||
deleted, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("count cleared SQL audit events: %w", err)
|
||||
}
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
// deleteTimestampPrefixLocked never removes an event after the first retained
|
||||
// sequence, even when a caller supplied an out-of-order timestamp. This keeps
|
||||
// every remaining adjacency and record hash untouched.
|
||||
func deleteTimestampPrefixLocked(conn *sql.Conn, beforeTimestamp int64) (int64, error) {
|
||||
var firstRetainedSequence int64
|
||||
err := conn.QueryRowContext(context.Background(), `SELECT sequence FROM sql_audit_events
|
||||
WHERE timestamp >= ? ORDER BY sequence ASC LIMIT 1`, beforeTimestamp).Scan(&firstRetainedSequence)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return deleteAllLocked(conn)
|
||||
}
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("locate SQL audit retention boundary: %w", err)
|
||||
}
|
||||
result, err := conn.ExecContext(context.Background(),
|
||||
`DELETE FROM sql_audit_events WHERE sequence < ?`, firstRetainedSequence)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("delete SQL audit sequence prefix: %w", err)
|
||||
}
|
||||
deleted, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("count deleted SQL audit sequence prefix: %w", err)
|
||||
}
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
type canonicalEvent struct {
|
||||
Version string `json:"version"`
|
||||
ID string `json:"id"`
|
||||
Sequence int64 `json:"sequence"`
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
EventType string `json:"eventType"`
|
||||
Status string `json:"status"`
|
||||
ConnectionID string `json:"connectionId"`
|
||||
ConnectionFingerprint string `json:"connectionFingerprint"`
|
||||
DBType string `json:"dbType"`
|
||||
Database string `json:"database"`
|
||||
QueryID string `json:"queryId"`
|
||||
TransactionID string `json:"transactionId"`
|
||||
Source string `json:"source"`
|
||||
BoundaryMode string `json:"boundaryMode"`
|
||||
CommitMode string `json:"commitMode"`
|
||||
SQLText string `json:"sqlText"`
|
||||
SQLRedacted bool `json:"sqlRedacted"`
|
||||
SQLFingerprint string `json:"sqlFingerprint"`
|
||||
StatementIndex int `json:"statementIndex"`
|
||||
StatementCount int `json:"statementCount"`
|
||||
DurationMs int64 `json:"durationMs"`
|
||||
RowsAffected int64 `json:"rowsAffected"`
|
||||
RowsReturned int64 `json:"rowsReturned"`
|
||||
Error string `json:"error"`
|
||||
PrevHash string `json:"prevHash"`
|
||||
}
|
||||
|
||||
func calculateEventHash(event Event) (string, error) {
|
||||
payload, err := json.Marshal(canonicalEvent{
|
||||
Version: chainVersion,
|
||||
ID: event.ID,
|
||||
Sequence: event.Sequence,
|
||||
Timestamp: event.Timestamp,
|
||||
EventType: event.EventType,
|
||||
Status: event.Status,
|
||||
ConnectionID: event.ConnectionID,
|
||||
ConnectionFingerprint: event.ConnectionFingerprint,
|
||||
DBType: event.DBType,
|
||||
Database: event.Database,
|
||||
QueryID: event.QueryID,
|
||||
TransactionID: event.TransactionID,
|
||||
Source: event.Source,
|
||||
BoundaryMode: event.BoundaryMode,
|
||||
CommitMode: event.CommitMode,
|
||||
SQLText: event.SQLText,
|
||||
SQLRedacted: event.SQLRedacted,
|
||||
SQLFingerprint: event.SQLFingerprint,
|
||||
StatementIndex: event.StatementIndex,
|
||||
StatementCount: event.StatementCount,
|
||||
DurationMs: event.DurationMs,
|
||||
RowsAffected: event.RowsAffected,
|
||||
RowsReturned: event.RowsReturned,
|
||||
Error: event.Error,
|
||||
PrevHash: event.PrevHash,
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("encode canonical SQL audit event: %w", err)
|
||||
}
|
||||
digest := sha256.Sum256(payload)
|
||||
return hex.EncodeToString(digest[:]), nil
|
||||
}
|
||||
|
||||
func invalidIntegrityReport(report IntegrityReport, sequence int64, message string) IntegrityReport {
|
||||
report.Valid = false
|
||||
report.InvalidSequence = sequence
|
||||
report.Message = message + " (unkeyed local hash chain; weak validation)"
|
||||
return report
|
||||
}
|
||||
|
||||
const selectEventColumns = `SELECT sequence, id, timestamp, event_type, status,
|
||||
connection_id, connection_fingerprint, db_type, database_name, query_id,
|
||||
transaction_id, source, boundary_mode, commit_mode, sql_text, sql_redacted,
|
||||
sql_fingerprint, statement_index, statement_count, duration_ms, rows_affected,
|
||||
rows_returned, error_text, prev_hash, record_hash FROM sql_audit_events`
|
||||
|
||||
type rowScanner interface {
|
||||
Scan(...any) error
|
||||
}
|
||||
|
||||
func scanEvent(scanner rowScanner) (Event, error) {
|
||||
var event Event
|
||||
var sqlRedacted int
|
||||
if err := scanner.Scan(
|
||||
&event.Sequence, &event.ID, &event.Timestamp, &event.EventType, &event.Status,
|
||||
&event.ConnectionID, &event.ConnectionFingerprint, &event.DBType, &event.Database,
|
||||
&event.QueryID, &event.TransactionID, &event.Source, &event.BoundaryMode,
|
||||
&event.CommitMode, &event.SQLText, &sqlRedacted, &event.SQLFingerprint,
|
||||
&event.StatementIndex, &event.StatementCount, &event.DurationMs,
|
||||
&event.RowsAffected, &event.RowsReturned, &event.Error, &event.PrevHash, &event.Hash,
|
||||
); err != nil {
|
||||
return Event{}, fmt.Errorf("scan SQL audit event: %w", err)
|
||||
}
|
||||
event.SQLRedacted = sqlRedacted != 0
|
||||
return event, nil
|
||||
}
|
||||
|
||||
func boolToInt(value bool) int {
|
||||
if value {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
652
internal/sqlaudit/store_test.go
Normal file
652
internal/sqlaudit/store_test.go
Normal file
@@ -0,0 +1,652 @@
|
||||
package sqlaudit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func openTestStore(t *testing.T) *Store {
|
||||
t.Helper()
|
||||
store, err := Open(filepath.Join(t.TempDir(), "audit", "sql_audit.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("Open returned error: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := store.Close(); err != nil && !errors.Is(err, ErrClosed) {
|
||||
t.Errorf("Close returned error: %v", err)
|
||||
}
|
||||
})
|
||||
return store
|
||||
}
|
||||
|
||||
func sampleEvent(id string, timestamp int64) Event {
|
||||
return Event{
|
||||
ID: id,
|
||||
Timestamp: timestamp,
|
||||
EventType: "query",
|
||||
Status: "success",
|
||||
ConnectionID: "conn-main",
|
||||
ConnectionFingerprint: "postgres://admin:raw-secret@db.example/app",
|
||||
DBType: "mysql",
|
||||
Database: "analytics",
|
||||
QueryID: "query-1",
|
||||
Source: "query_editor",
|
||||
BoundaryMode: BoundaryModeDriverAPI,
|
||||
SQLText: "SELECT * FROM users WHERE id = 42 AND token = 'raw-token'",
|
||||
DurationMs: 25,
|
||||
RowsReturned: 1,
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenConfiguresSQLiteAndDefaultSettings(t *testing.T) {
|
||||
store := openTestStore(t)
|
||||
settings, err := store.GetSettings()
|
||||
if err != nil {
|
||||
t.Fatalf("GetSettings returned error: %v", err)
|
||||
}
|
||||
if settings != DefaultSettings() {
|
||||
t.Fatalf("unexpected defaults: %#v", settings)
|
||||
}
|
||||
|
||||
var journalMode string
|
||||
if err := store.db.QueryRow(`PRAGMA journal_mode`).Scan(&journalMode); err != nil {
|
||||
t.Fatalf("read journal mode: %v", err)
|
||||
}
|
||||
if strings.ToLower(journalMode) != "wal" {
|
||||
t.Fatalf("journal mode must be WAL, got %q", journalMode)
|
||||
}
|
||||
var synchronous int
|
||||
if err := store.db.QueryRow(`PRAGMA synchronous`).Scan(&synchronous); err != nil {
|
||||
t.Fatalf("read synchronous mode: %v", err)
|
||||
}
|
||||
if synchronous != 2 {
|
||||
t.Fatalf("synchronous must be FULL (2), got %d", synchronous)
|
||||
}
|
||||
if got := store.db.Stats().MaxOpenConnections; got != 4 {
|
||||
t.Fatalf("MaxOpenConnections=%d, want 4", got)
|
||||
}
|
||||
|
||||
connections := make([]interface{ Close() error }, 0, 4)
|
||||
for index := 0; index < 4; index++ {
|
||||
conn, err := store.db.Conn(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("acquire pooled connection %d: %v", index+1, err)
|
||||
}
|
||||
connections = append(connections, conn)
|
||||
var busyTimeout, foreignKeys, connectionSynchronous int
|
||||
var connectionJournal string
|
||||
if err := conn.QueryRowContext(context.Background(), `PRAGMA busy_timeout`).Scan(&busyTimeout); err != nil {
|
||||
t.Fatalf("read connection %d busy_timeout: %v", index+1, err)
|
||||
}
|
||||
if err := conn.QueryRowContext(context.Background(), `PRAGMA foreign_keys`).Scan(&foreignKeys); err != nil {
|
||||
t.Fatalf("read connection %d foreign_keys: %v", index+1, err)
|
||||
}
|
||||
if err := conn.QueryRowContext(context.Background(), `PRAGMA synchronous`).Scan(&connectionSynchronous); err != nil {
|
||||
t.Fatalf("read connection %d synchronous: %v", index+1, err)
|
||||
}
|
||||
if err := conn.QueryRowContext(context.Background(), `PRAGMA journal_mode`).Scan(&connectionJournal); err != nil {
|
||||
t.Fatalf("read connection %d journal_mode: %v", index+1, err)
|
||||
}
|
||||
if busyTimeout != 5000 || foreignKeys != 1 || connectionSynchronous != 2 || strings.ToLower(connectionJournal) != "wal" {
|
||||
t.Fatalf("connection %d PRAGMAs unexpected: busy=%d foreign=%d sync=%d journal=%q",
|
||||
index+1, busyTimeout, foreignKeys, connectionSynchronous, connectionJournal)
|
||||
}
|
||||
}
|
||||
for _, conn := range connections {
|
||||
if err := conn.Close(); err != nil {
|
||||
t.Fatalf("close pooled connection: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if runtime.GOOS != "windows" {
|
||||
dirInfo, err := os.Stat(filepath.Dir(store.Path()))
|
||||
if err != nil {
|
||||
t.Fatalf("stat audit directory: %v", err)
|
||||
}
|
||||
if got := dirInfo.Mode().Perm(); got != 0o700 {
|
||||
t.Fatalf("audit directory mode = %#o, want 0700", got)
|
||||
}
|
||||
fileInfo, err := os.Stat(store.Path())
|
||||
if err != nil {
|
||||
t.Fatalf("stat audit database: %v", err)
|
||||
}
|
||||
if got := fileInfo.Mode().Perm(); got != 0o600 {
|
||||
t.Fatalf("audit database mode = %#o, want 0600", got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenSupportsURIReservedCharactersInPath(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "audit space # percent% amp&", "sql audit.db")
|
||||
store, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Open reserved-character path returned error: %v", err)
|
||||
}
|
||||
if err := store.Append(sampleEvent("reserved-path", time.Now().UnixMilli())); err != nil {
|
||||
_ = store.Close()
|
||||
t.Fatalf("Append reserved-character path returned error: %v", err)
|
||||
}
|
||||
if err := store.Close(); err != nil {
|
||||
t.Fatalf("Close reserved-character path returned error: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Fatalf("SQLite database was not created at literal path %q: %v", path, err)
|
||||
}
|
||||
if info, err := os.Stat(path + "-wal"); err == nil && info.Size() > 0 {
|
||||
t.Fatalf("Close should checkpoint/truncate WAL before migration, size=%d", info.Size())
|
||||
} else if err != nil && !os.IsNotExist(err) {
|
||||
t.Fatalf("stat WAL after Close: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendSanitizesAndQueryFiltersWithSummary(t *testing.T) {
|
||||
store := openTestStore(t)
|
||||
now := time.Now().UnixMilli()
|
||||
first := sampleEvent("event-1", now-100)
|
||||
first.TransactionID = "tx-1"
|
||||
if err := store.Append(first); err != nil {
|
||||
t.Fatalf("Append first returned error: %v", err)
|
||||
}
|
||||
second := sampleEvent("event-2", now)
|
||||
second.Status = "error"
|
||||
second.TransactionID = "tx-2"
|
||||
second.Error = "password=driver-secret failed near 'raw-token'"
|
||||
if err := store.Append(second); err != nil {
|
||||
t.Fatalf("Append second returned error: %v", err)
|
||||
}
|
||||
|
||||
page, err := store.Query(Filter{Database: "analytics", Page: 1, PageSize: 1})
|
||||
if err != nil {
|
||||
t.Fatalf("Query returned error: %v", err)
|
||||
}
|
||||
if page.Total != 2 || len(page.Items) != 1 || page.Items[0].ID != "event-2" {
|
||||
t.Fatalf("unexpected page: %#v", page)
|
||||
}
|
||||
if page.Summary.TotalEvents != 2 || page.Summary.SuccessCount != 1 ||
|
||||
page.Summary.ErrorCount != 1 || page.Summary.TransactionCount != 2 {
|
||||
t.Fatalf("unexpected summary: %#v", page.Summary)
|
||||
}
|
||||
event := page.Items[0]
|
||||
for _, secret := range []string{"42", "raw-token", "driver-secret", "admin", "raw-secret", "db.example"} {
|
||||
if strings.Contains(event.SQLText+event.Error+event.ConnectionFingerprint, secret) {
|
||||
t.Fatalf("stored audit event leaked %q: %#v", secret, event)
|
||||
}
|
||||
}
|
||||
if !event.SQLRedacted || event.SQLFingerprint == "" || len(event.ConnectionFingerprint) != 64 {
|
||||
t.Fatalf("event was not safely normalized: %#v", event)
|
||||
}
|
||||
|
||||
errorPage, err := store.Query(Filter{Status: "error", Search: "password", PageSize: 10})
|
||||
if err != nil || errorPage.Total != 1 || len(errorPage.Items) != 1 {
|
||||
t.Fatalf("filtered query failed: page=%#v err=%v", errorPage, err)
|
||||
}
|
||||
|
||||
assertAuditStorageFilesDoNotContain(t, store.Path(), []string{"raw-token", "driver-secret", "raw-secret"})
|
||||
}
|
||||
|
||||
func TestQuerySupportsContractFiltersAndEscapesSearchWildcards(t *testing.T) {
|
||||
store := openTestStore(t)
|
||||
now := time.Now().UnixMilli()
|
||||
target := sampleEvent("filter-target", now)
|
||||
target.EventType = "TRANSACTION_COMMIT"
|
||||
target.Status = "SUCCESS"
|
||||
target.TransactionID = "tx-filter"
|
||||
target.Source = "SYSTEM"
|
||||
target.DBType = "MYSQL"
|
||||
if err := store.Append(target); err != nil {
|
||||
t.Fatalf("Append target returned error: %v", err)
|
||||
}
|
||||
other := sampleEvent("filter-other", now+100)
|
||||
other.ConnectionID = "conn-other"
|
||||
other.Database = "reporting"
|
||||
other.TransactionID = "tx-other"
|
||||
if err := store.Append(other); err != nil {
|
||||
t.Fatalf("Append other returned error: %v", err)
|
||||
}
|
||||
|
||||
page, err := store.Query(Filter{
|
||||
ConnectionID: "conn-main",
|
||||
Database: "analytics",
|
||||
DBType: "MYSQL",
|
||||
EventType: "TRANSACTION_COMMIT",
|
||||
Status: "SUCCESS",
|
||||
TransactionID: "tx-filter",
|
||||
Source: "SYSTEM",
|
||||
FromTimestamp: now - 1,
|
||||
ToTimestamp: now + 1,
|
||||
PageSize: 10,
|
||||
})
|
||||
if err != nil || page.Total != 1 || len(page.Items) != 1 || page.Items[0].ID != "filter-target" {
|
||||
t.Fatalf("contract filters did not isolate target: page=%#v err=%v", page, err)
|
||||
}
|
||||
|
||||
page, err = store.Query(Filter{Search: `%_' OR 1=1 --`, PageSize: 10})
|
||||
if err != nil || page.Total != 0 {
|
||||
t.Fatalf("literal wildcard/injection-like search should not broaden results: page=%#v err=%v", page, err)
|
||||
}
|
||||
|
||||
all, err := store.Query(Filter{PageSize: 10})
|
||||
if err != nil || len(all.Items) == 0 {
|
||||
t.Fatalf("load events for fingerprint search: page=%#v err=%v", all, err)
|
||||
}
|
||||
fingerprintPrefix := all.Items[0].SQLFingerprint[:12]
|
||||
page, err = store.Query(Filter{Search: fingerprintPrefix, PageSize: 10})
|
||||
if err != nil || page.Total == 0 {
|
||||
t.Fatalf("fingerprint search should find matching events: page=%#v err=%v", page, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataAndDisabledCaptureModes(t *testing.T) {
|
||||
store := openTestStore(t)
|
||||
settings := DefaultSettings()
|
||||
settings.CaptureMode = CaptureModeMetadata
|
||||
if err := store.UpdateSettings(settings); err != nil {
|
||||
t.Fatalf("UpdateSettings(metadata) returned error: %v", err)
|
||||
}
|
||||
if err := store.Append(sampleEvent("metadata-event", time.Now().UnixMilli())); err != nil {
|
||||
t.Fatalf("Append metadata event returned error: %v", err)
|
||||
}
|
||||
page, err := store.Query(Filter{PageSize: 10})
|
||||
if err != nil || len(page.Items) != 1 {
|
||||
t.Fatalf("query metadata event: page=%#v err=%v", page, err)
|
||||
}
|
||||
if page.Items[0].SQLText != "" || page.Items[0].SQLFingerprint == "" || !page.Items[0].SQLRedacted {
|
||||
t.Fatalf("metadata capture persisted SQL text: %#v", page.Items[0])
|
||||
}
|
||||
|
||||
settings.Enabled = false
|
||||
if err := store.UpdateSettings(settings); err != nil {
|
||||
t.Fatalf("UpdateSettings(disabled) returned error: %v", err)
|
||||
}
|
||||
if err := store.Append(sampleEvent("disabled-event", time.Now().UnixMilli())); err != nil {
|
||||
t.Fatalf("disabled Append should be a no-op: %v", err)
|
||||
}
|
||||
page, err = store.Query(Filter{PageSize: 10})
|
||||
if err != nil || page.Total != 1 {
|
||||
t.Fatalf("disabled capture should not append: page=%#v err=%v", page, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestControlBoundariesAreAtomicAndBypassDisabledCapture(t *testing.T) {
|
||||
store := openTestStore(t)
|
||||
settings := DefaultSettings()
|
||||
settings.Enabled = false
|
||||
settings.CaptureMode = CaptureModeMetadata
|
||||
control := Event{ID: "settings-control", EventType: "audit_settings_change", Status: "success", Source: "audit_control"}
|
||||
if err := store.UpdateSettingsWithControl(settings, control); err != nil {
|
||||
t.Fatalf("UpdateSettingsWithControl returned error: %v", err)
|
||||
}
|
||||
if err := store.Append(sampleEvent("disabled-ordinary", time.Now().UnixMilli())); err != nil {
|
||||
t.Fatalf("disabled ordinary append returned error: %v", err)
|
||||
}
|
||||
page, err := store.Query(Filter{PageSize: 10})
|
||||
if err != nil || page.Total != 1 || page.Items[0].ID != "settings-control" {
|
||||
t.Fatalf("disabled control boundary was not retained: page=%#v err=%v", page, err)
|
||||
}
|
||||
if !strings.Contains(page.Items[0].SQLText, "FROM_ENABLED_ON TO_ENABLED_OFF") {
|
||||
t.Fatalf("settings control boundary lacks safe old/new metadata: %q", page.Items[0].SQLText)
|
||||
}
|
||||
|
||||
deleted, err := store.ClearWithControl(0, Event{ID: "clear-control", EventType: "audit_clear", Status: "success", Source: "audit_control"})
|
||||
if err != nil || deleted != 1 {
|
||||
t.Fatalf("ClearWithControl deleted=%d err=%v, want 1", deleted, err)
|
||||
}
|
||||
page, err = store.Query(Filter{PageSize: 10})
|
||||
if err != nil || page.Total != 1 || page.Items[0].ID != "clear-control" || page.Items[0].RowsAffected != 1 {
|
||||
t.Fatalf("clear control boundary was not atomically retained: page=%#v err=%v", page, err)
|
||||
}
|
||||
if page.Items[0].SQLText != "AUDIT CLEAR ALL" {
|
||||
t.Fatalf("metadata capture erased the clear control descriptor: %q", page.Items[0].SQLText)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendNeverPersistsRedisOrMessagePayloadSecrets(t *testing.T) {
|
||||
store := openTestStore(t)
|
||||
now := time.Now().UnixMilli()
|
||||
redisEvent := sampleEvent("redis-secret", now)
|
||||
redisEvent.DBType = "redis"
|
||||
redisEvent.SQLText = `HSET account:42 email private@example.test token redis-token-secret`
|
||||
redisEvent.Error = `AUTH failed password=redis-error-secret`
|
||||
if err := store.Append(redisEvent); err != nil {
|
||||
t.Fatalf("Append Redis event returned error: %v", err)
|
||||
}
|
||||
mqEvent := sampleEvent("mq-secret", now+1)
|
||||
mqEvent.DBType = "rocketmq"
|
||||
mqEvent.SQLText = `PUBLISH orders {"token":"mq-payload-secret","card":"4111111111111111"}`
|
||||
mqEvent.Error = `publish rejected token=mq-error-secret`
|
||||
if err := store.Append(mqEvent); err != nil {
|
||||
t.Fatalf("Append MQ event returned error: %v", err)
|
||||
}
|
||||
mqEventSecond := sampleEvent("mq-secret-second", now+2)
|
||||
mqEventSecond.DBType = "rocketmq"
|
||||
mqEventSecond.SQLText = `PUBLISH orders {"token":"different-low-entropy-secret"}`
|
||||
if err := store.Append(mqEventSecond); err != nil {
|
||||
t.Fatalf("Append second MQ event returned error: %v", err)
|
||||
}
|
||||
|
||||
page, err := store.Query(Filter{PageSize: 10})
|
||||
if err != nil || len(page.Items) != 3 {
|
||||
t.Fatalf("query non-SQL audit events: page=%#v err=%v", page, err)
|
||||
}
|
||||
byID := map[string]Event{}
|
||||
for _, event := range page.Items {
|
||||
byID[event.ID] = event
|
||||
}
|
||||
if got := byID["redis-secret"].SQLText; got != "HSET account:42 email ? token ?" {
|
||||
t.Fatalf("unexpected Redis audit text: %q", got)
|
||||
}
|
||||
if got := byID["mq-secret"].SQLText; got != "" {
|
||||
t.Fatalf("message payload must be metadata-only, got %q", got)
|
||||
}
|
||||
if byID["mq-secret"].SQLFingerprint == "" {
|
||||
t.Fatal("metadata-only message event still needs a safe fingerprint")
|
||||
}
|
||||
if byID["mq-secret"].SQLFingerprint != byID["mq-secret-second"].SQLFingerprint {
|
||||
t.Fatalf("metadata-only fingerprint must not depend on message payload: first=%s second=%s",
|
||||
byID["mq-secret"].SQLFingerprint, byID["mq-secret-second"].SQLFingerprint)
|
||||
}
|
||||
|
||||
assertAuditStorageFilesDoNotContain(t, store.Path(), []string{
|
||||
"private@example.test", "redis-token-secret", "redis-error-secret",
|
||||
"mq-payload-secret", "4111111111111111", "mq-error-secret", "different-low-entropy-secret",
|
||||
})
|
||||
}
|
||||
|
||||
func TestIntegrityDetectsTampering(t *testing.T) {
|
||||
store := openTestStore(t)
|
||||
now := time.Now().UnixMilli()
|
||||
for index := 0; index < 2; index++ {
|
||||
if err := store.Append(sampleEvent("integrity-"+string(rune('a'+index)), now+int64(index))); err != nil {
|
||||
t.Fatalf("Append returned error: %v", err)
|
||||
}
|
||||
}
|
||||
report, err := store.VerifyIntegrity()
|
||||
if err != nil || !report.Valid || !report.WeakValidation || report.CheckedRecords != 2 {
|
||||
t.Fatalf("unexpected valid integrity report: %#v err=%v", report, err)
|
||||
}
|
||||
if _, err := store.db.Exec(`UPDATE sql_audit_events SET sql_text='tampered' WHERE sequence=1`); err != nil {
|
||||
t.Fatalf("tamper test database: %v", err)
|
||||
}
|
||||
report, err = store.VerifyIntegrity()
|
||||
if err != nil || report.Valid || report.InvalidSequence != 1 {
|
||||
t.Fatalf("tampering was not detected: %#v err=%v", report, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClearAndRecordLimitPreserveRemainingHashes(t *testing.T) {
|
||||
store := openTestStore(t)
|
||||
now := time.Now().UnixMilli()
|
||||
for index := 0; index < 3; index++ {
|
||||
if err := store.Append(sampleEvent("clear-"+string(rune('a'+index)), now-3000+int64(index*1000))); err != nil {
|
||||
t.Fatalf("Append returned error: %v", err)
|
||||
}
|
||||
}
|
||||
beforeClear, err := store.Query(Filter{PageSize: 10})
|
||||
if err != nil || len(beforeClear.Items) != 3 {
|
||||
t.Fatalf("query before clear: page=%#v err=%v", beforeClear, err)
|
||||
}
|
||||
retainedHash := beforeClear.Items[0].Hash
|
||||
deleted, err := store.Clear(now - 1500)
|
||||
if err != nil || deleted != 2 {
|
||||
t.Fatalf("Clear deleted=%d err=%v, want 2", deleted, err)
|
||||
}
|
||||
page, err := store.Query(Filter{PageSize: 10})
|
||||
if err != nil || len(page.Items) != 1 || page.Items[0].PrevHash == "" {
|
||||
t.Fatalf("clear rewrote or lost the retained chain anchor: page=%#v err=%v", page, err)
|
||||
}
|
||||
if page.Items[0].Hash != retainedHash {
|
||||
t.Fatalf("clear must not rewrite retained event hash: before=%s after=%s", retainedHash, page.Items[0].Hash)
|
||||
}
|
||||
if report, err := store.VerifyIntegrity(); err != nil || !report.Valid || !report.PartialChain || !report.TruncatedPrefix {
|
||||
t.Fatalf("chain invalid after clear: report=%#v err=%v", report, err)
|
||||
}
|
||||
|
||||
settings := DefaultSettings()
|
||||
settings.MaxRecords = 2
|
||||
if err := store.UpdateSettings(settings); err != nil {
|
||||
t.Fatalf("UpdateSettings returned error: %v", err)
|
||||
}
|
||||
for index := 0; index < 3; index++ {
|
||||
if err := store.Append(sampleEvent("limited-"+string(rune('a'+index)), now+int64(index))); err != nil {
|
||||
t.Fatalf("Append limited event returned error: %v", err)
|
||||
}
|
||||
}
|
||||
page, err = store.Query(Filter{PageSize: 10})
|
||||
if err != nil || page.Total != 2 {
|
||||
t.Fatalf("record limit not enforced: page=%#v err=%v", page, err)
|
||||
}
|
||||
if report, err := store.VerifyIntegrity(); err != nil || !report.Valid || !report.PartialChain {
|
||||
t.Fatalf("chain invalid after record-limit pruning: report=%#v err=%v", report, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClearDeletesOnlyContiguousPrefixForOutOfOrderTimestamps(t *testing.T) {
|
||||
store := openTestStore(t)
|
||||
now := time.Now().UnixMilli()
|
||||
for _, event := range []Event{
|
||||
sampleEvent("prefix-old", now-10_000),
|
||||
sampleEvent("prefix-new", now),
|
||||
sampleEvent("later-sequence-old-time", now-20_000),
|
||||
} {
|
||||
if err := store.Append(event); err != nil {
|
||||
t.Fatalf("Append returned error: %v", err)
|
||||
}
|
||||
}
|
||||
before, err := store.Query(Filter{PageSize: 10})
|
||||
if err != nil || len(before.Items) != 3 {
|
||||
t.Fatalf("query before clear: page=%#v err=%v", before, err)
|
||||
}
|
||||
hashes := map[string]string{}
|
||||
for _, event := range before.Items {
|
||||
hashes[event.ID] = event.Hash
|
||||
}
|
||||
|
||||
deleted, err := store.Clear(now - 5_000)
|
||||
if err != nil || deleted != 1 {
|
||||
t.Fatalf("Clear should delete only one contiguous prefix event: deleted=%d err=%v", deleted, err)
|
||||
}
|
||||
after, err := store.Query(Filter{PageSize: 10})
|
||||
if err != nil || len(after.Items) != 2 {
|
||||
t.Fatalf("query after clear: page=%#v err=%v", after, err)
|
||||
}
|
||||
for _, event := range after.Items {
|
||||
if event.Hash != hashes[event.ID] {
|
||||
t.Fatalf("retained event %s hash was rewritten", event.ID)
|
||||
}
|
||||
}
|
||||
if report, err := store.VerifyIntegrity(); err != nil || !report.Valid || !report.PartialChain {
|
||||
t.Fatalf("partial chain should remain verifiable: report=%#v err=%v", report, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrentAppendKeepsUniqueOrderedChain(t *testing.T) {
|
||||
store := openTestStore(t)
|
||||
const count = 32
|
||||
now := time.Now().UnixMilli()
|
||||
var wait sync.WaitGroup
|
||||
errorsCh := make(chan error, count)
|
||||
for index := 0; index < count; index++ {
|
||||
wait.Add(1)
|
||||
go func(index int) {
|
||||
defer wait.Done()
|
||||
event := sampleEvent("concurrent-"+time.UnixMilli(int64(index)).Format("150405.000000000"), now+int64(index))
|
||||
errorsCh <- store.Append(event)
|
||||
}(index)
|
||||
}
|
||||
wait.Wait()
|
||||
close(errorsCh)
|
||||
for err := range errorsCh {
|
||||
if err != nil {
|
||||
t.Fatalf("concurrent Append returned error: %v", err)
|
||||
}
|
||||
}
|
||||
page, err := store.Query(Filter{PageSize: count})
|
||||
if err != nil || page.Total != count {
|
||||
t.Fatalf("concurrent events missing: total=%d err=%v", page.Total, err)
|
||||
}
|
||||
seen := make(map[int64]struct{}, count)
|
||||
for _, event := range page.Items {
|
||||
if _, duplicate := seen[event.Sequence]; duplicate {
|
||||
t.Fatalf("duplicate sequence %d", event.Sequence)
|
||||
}
|
||||
seen[event.Sequence] = struct{}{}
|
||||
}
|
||||
if report, err := store.VerifyIntegrity(); err != nil || !report.Valid || report.CheckedRecords != count {
|
||||
t.Fatalf("concurrent chain invalid: report=%#v err=%v", report, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWALReaderDoesNotBlockSynchronousAppend(t *testing.T) {
|
||||
store := openTestStore(t)
|
||||
now := time.Now().UnixMilli()
|
||||
if err := store.Append(sampleEvent("wal-seed", now)); err != nil {
|
||||
t.Fatalf("Append seed returned error: %v", err)
|
||||
}
|
||||
reader, err := store.db.Conn(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("acquire reader connection: %v", err)
|
||||
}
|
||||
defer reader.Close()
|
||||
if _, err := reader.ExecContext(context.Background(), `BEGIN`); err != nil {
|
||||
t.Fatalf("begin reader transaction: %v", err)
|
||||
}
|
||||
rows, err := reader.QueryContext(context.Background(), `SELECT id FROM sql_audit_events ORDER BY sequence`)
|
||||
if err != nil {
|
||||
_, _ = reader.ExecContext(context.Background(), `ROLLBACK`)
|
||||
t.Fatalf("open reader cursor: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
if !rows.Next() {
|
||||
t.Fatal("reader cursor should expose seed event")
|
||||
}
|
||||
var seedID string
|
||||
if err := rows.Scan(&seedID); err != nil || seedID != "wal-seed" {
|
||||
t.Fatalf("scan reader cursor id=%q err=%v", seedID, err)
|
||||
}
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- store.Append(sampleEvent("wal-writer", now+1))
|
||||
}()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatalf("Append while WAL reader active returned error: %v", err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("active WAL reader blocked synchronous audit append")
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
t.Fatalf("close reader rows: %v", err)
|
||||
}
|
||||
if _, err := reader.ExecContext(context.Background(), `ROLLBACK`); err != nil {
|
||||
t.Fatalf("rollback reader transaction: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendBatchIsAtomicAndKeepsOneContinuousChain(t *testing.T) {
|
||||
store := openTestStore(t)
|
||||
now := time.Now().UnixMilli()
|
||||
if err := store.Append(sampleEvent("before-batch", now)); err != nil {
|
||||
t.Fatalf("Append before batch returned error: %v", err)
|
||||
}
|
||||
batch := []Event{
|
||||
sampleEvent("batch-1", now+1),
|
||||
sampleEvent("batch-2", now+2),
|
||||
sampleEvent("batch-3", now+3),
|
||||
}
|
||||
if err := store.AppendBatch(batch); err != nil {
|
||||
t.Fatalf("AppendBatch returned error: %v", err)
|
||||
}
|
||||
page, err := store.Query(Filter{PageSize: 10})
|
||||
if err != nil || page.Total != 4 {
|
||||
t.Fatalf("query after batch: page=%#v err=%v", page, err)
|
||||
}
|
||||
report, err := store.VerifyIntegrity()
|
||||
if err != nil || !report.Valid || report.CheckedRecords != 4 {
|
||||
t.Fatalf("integrity after batch: report=%#v err=%v", report, err)
|
||||
}
|
||||
|
||||
duplicateBatch := []Event{
|
||||
sampleEvent("atomic-new", now+4),
|
||||
sampleEvent("batch-2", now+5),
|
||||
}
|
||||
if err := store.AppendBatch(duplicateBatch); err == nil {
|
||||
t.Fatal("AppendBatch with duplicate ID should fail")
|
||||
}
|
||||
page, err = store.Query(Filter{Search: "atomic-new", PageSize: 10})
|
||||
if err != nil || page.Total != 0 {
|
||||
t.Fatalf("failed batch should roll back every event: page=%#v err=%v", page, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendBatchLargerThanMaxRecordsFailsAtomically(t *testing.T) {
|
||||
store := openTestStore(t)
|
||||
settings := DefaultSettings()
|
||||
settings.MaxRecords = 3
|
||||
if err := store.UpdateSettings(settings); err != nil {
|
||||
t.Fatalf("UpdateSettings returned error: %v", err)
|
||||
}
|
||||
now := time.Now().UnixMilli()
|
||||
if err := store.Append(sampleEvent("existing-before-large-batch", now)); err != nil {
|
||||
t.Fatalf("Append existing event returned error: %v", err)
|
||||
}
|
||||
batch := make([]Event, 0, 6)
|
||||
for index := 0; index < 6; index++ {
|
||||
batch = append(batch, sampleEvent(fmt.Sprintf("large-batch-%d", index), now+int64(index+1)))
|
||||
}
|
||||
if err := store.AppendBatch(batch); !errors.Is(err, ErrBatchExceedsMaxRecords) {
|
||||
t.Fatalf("AppendBatch error = %v, want ErrBatchExceedsMaxRecords", err)
|
||||
}
|
||||
page, err := store.Query(Filter{PageSize: 10})
|
||||
if err != nil {
|
||||
t.Fatalf("Query returned error: %v", err)
|
||||
}
|
||||
if page.Total != 1 || len(page.Items) != 1 || page.Items[0].ID != "existing-before-large-batch" {
|
||||
t.Fatalf("oversized batch must not modify existing audit records: %#v", page)
|
||||
}
|
||||
if report, err := store.VerifyIntegrity(); err != nil || !report.Valid || report.CheckedRecords != 1 {
|
||||
t.Fatalf("large-batch chain invalid: report=%#v err=%v", report, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSettingsValidationAndClosedStore(t *testing.T) {
|
||||
store := openTestStore(t)
|
||||
invalid := DefaultSettings()
|
||||
invalid.CaptureMode = "full"
|
||||
if err := store.UpdateSettings(invalid); err == nil {
|
||||
t.Fatal("full/raw SQL capture mode must be rejected")
|
||||
}
|
||||
if err := store.Close(); err != nil {
|
||||
t.Fatalf("Close returned error: %v", err)
|
||||
}
|
||||
if err := store.Append(sampleEvent("closed", time.Now().UnixMilli())); !errors.Is(err, ErrClosed) {
|
||||
t.Fatalf("Append after Close error=%v, want ErrClosed", err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertAuditStorageFilesDoNotContain(t *testing.T, databasePath string, secrets []string) {
|
||||
t.Helper()
|
||||
for _, path := range []string{databasePath, databasePath + "-wal", databasePath + "-shm"} {
|
||||
payload, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
t.Fatalf("read SQLite audit storage file %q: %v", path, err)
|
||||
}
|
||||
for _, secret := range secrets {
|
||||
if strings.Contains(string(payload), secret) {
|
||||
t.Fatalf("SQLite audit storage file %q contains raw secret %q", path, secret)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
114
internal/sqlaudit/types.go
Normal file
114
internal/sqlaudit/types.go
Normal file
@@ -0,0 +1,114 @@
|
||||
package sqlaudit
|
||||
|
||||
const (
|
||||
CaptureModeRedacted = "redacted"
|
||||
CaptureModeMetadata = "metadata"
|
||||
|
||||
BoundaryModeDriverAPI = "driver_api"
|
||||
BoundaryModeTextSQL = "text_sql"
|
||||
BoundaryModeImplicit = "implicit"
|
||||
BoundaryModeUnknown = "unknown"
|
||||
|
||||
CommitModeAuto = "auto"
|
||||
CommitModeManual = "manual"
|
||||
CommitModePending = "pending"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultPageSize = 50
|
||||
maxPageSize = 500
|
||||
defaultRetentionDays = 30
|
||||
defaultMaxRecords = 100_000
|
||||
)
|
||||
|
||||
// Event is one immutable SQL audit fact. SQLText is always sanitized again by
|
||||
// Store.Append; callers cannot use this type to persist raw SQL.
|
||||
type Event struct {
|
||||
ID string `json:"id"`
|
||||
Sequence int64 `json:"sequence"`
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
EventType string `json:"eventType"`
|
||||
Status string `json:"status"`
|
||||
ConnectionID string `json:"connectionId"`
|
||||
ConnectionFingerprint string `json:"connectionFingerprint"`
|
||||
DBType string `json:"dbType"`
|
||||
Database string `json:"database"`
|
||||
QueryID string `json:"queryId"`
|
||||
TransactionID string `json:"transactionId"`
|
||||
Source string `json:"source"`
|
||||
BoundaryMode string `json:"boundaryMode"`
|
||||
CommitMode string `json:"commitMode"`
|
||||
SQLText string `json:"sqlText"`
|
||||
SQLRedacted bool `json:"sqlRedacted"`
|
||||
SQLFingerprint string `json:"sqlFingerprint"`
|
||||
StatementIndex int `json:"statementIndex"`
|
||||
StatementCount int `json:"statementCount"`
|
||||
DurationMs int64 `json:"durationMs"`
|
||||
RowsAffected int64 `json:"rowsAffected"`
|
||||
RowsReturned int64 `json:"rowsReturned"`
|
||||
Error string `json:"error"`
|
||||
PrevHash string `json:"prevHash"`
|
||||
Hash string `json:"hash"`
|
||||
}
|
||||
|
||||
type Filter struct {
|
||||
Search string `json:"search"`
|
||||
ConnectionID string `json:"connectionId"`
|
||||
Database string `json:"database"`
|
||||
DBType string `json:"dbType"`
|
||||
EventType string `json:"eventType"`
|
||||
Status string `json:"status"`
|
||||
TransactionID string `json:"transactionId"`
|
||||
Source string `json:"source"`
|
||||
FromTimestamp int64 `json:"fromTimestamp"`
|
||||
ToTimestamp int64 `json:"toTimestamp"`
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"pageSize"`
|
||||
}
|
||||
|
||||
type Summary struct {
|
||||
TotalEvents int64 `json:"totalEvents"`
|
||||
SuccessCount int64 `json:"successCount"`
|
||||
ErrorCount int64 `json:"errorCount"`
|
||||
TransactionCount int64 `json:"transactionCount"`
|
||||
}
|
||||
|
||||
type Page struct {
|
||||
Items []Event `json:"items"`
|
||||
Total int64 `json:"total"`
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"pageSize"`
|
||||
Summary Summary `json:"summary"`
|
||||
}
|
||||
|
||||
type Settings struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
CaptureMode string `json:"captureMode"`
|
||||
RetentionDays int `json:"retentionDays"`
|
||||
MaxRecords int `json:"maxRecords"`
|
||||
}
|
||||
|
||||
func DefaultSettings() Settings {
|
||||
return Settings{
|
||||
Enabled: true,
|
||||
CaptureMode: CaptureModeRedacted,
|
||||
RetentionDays: defaultRetentionDays,
|
||||
MaxRecords: defaultMaxRecords,
|
||||
}
|
||||
}
|
||||
|
||||
// IntegrityReport reports local SHA-256 chain consistency. WeakValidation is
|
||||
// always true because an unkeyed chain detects accidental/local edits but does
|
||||
// not stop an attacker who can rewrite both records and hashes.
|
||||
type IntegrityReport struct {
|
||||
Valid bool `json:"valid"`
|
||||
WeakValidation bool `json:"weakValidation"`
|
||||
PartialChain bool `json:"partialChain"`
|
||||
TruncatedPrefix bool `json:"truncatedPrefix"`
|
||||
Algorithm string `json:"algorithm"`
|
||||
CheckedRecords int64 `json:"checkedRecords"`
|
||||
FirstSequence int64 `json:"firstSequence,omitempty"`
|
||||
LastSequence int64 `json:"lastSequence,omitempty"`
|
||||
InvalidSequence int64 `json:"invalidSequence,omitempty"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
Reference in New Issue
Block a user