mirror of
https://github.com/Syngnat/GoNavi.git
synced 2026-08-11 09:13:36 +08:00
✨ feat(synccdc): 新增 MongoDB 变更流适配器
- 建立 CDC 适配器注册、探测与安全游标契约 - 支持 MongoDB change stream、恢复令牌和批量读取 - 覆盖权限探测、取消关闭与流位置恢复测试
This commit is contained in:
Syngnat
95
internal/synccdc/adapter.go
Normal file
95
internal/synccdc/adapter.go
Normal file
@@ -0,0 +1,95 @@
|
||||
package synccdc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"GoNavi-Wails/internal/connection"
|
||||
)
|
||||
|
||||
type Position struct {
|
||||
Adapter string `json:"adapter"`
|
||||
Opaque json.RawMessage `json:"opaque"`
|
||||
}
|
||||
|
||||
type Barrier struct {
|
||||
Position Position `json:"position"`
|
||||
SnapshotToken json.RawMessage `json:"snapshotToken,omitempty"`
|
||||
}
|
||||
|
||||
type ObjectRef struct {
|
||||
Database string `json:"database,omitempty"`
|
||||
Schema string `json:"schema,omitempty"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
type Event struct {
|
||||
Object ObjectRef `json:"object"`
|
||||
Operation string `json:"operation"`
|
||||
Key map[string]interface{} `json:"key,omitempty"`
|
||||
Before map[string]interface{} `json:"before,omitempty"`
|
||||
After map[string]interface{} `json:"after,omitempty"`
|
||||
CommitTime time.Time `json:"commitTime"`
|
||||
SourceTxID string `json:"sourceTxId,omitempty"`
|
||||
}
|
||||
|
||||
type Transaction struct {
|
||||
Events []Event `json:"events"`
|
||||
Position Position `json:"position"`
|
||||
}
|
||||
|
||||
type Request struct {
|
||||
Config connection.ConnectionConfig `json:"-"`
|
||||
Objects []ObjectRef `json:"objects"`
|
||||
Database string `json:"database,omitempty"`
|
||||
Schema string `json:"schema,omitempty"`
|
||||
}
|
||||
|
||||
type Capability struct {
|
||||
Adapter string `json:"adapter"`
|
||||
SourceType string `json:"sourceType"`
|
||||
Supported bool `json:"supported"`
|
||||
Ready bool `json:"ready"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
RequiredSettings []string `json:"requiredSettings,omitempty"`
|
||||
SupportsInitialSnapshot bool `json:"supportsInitialSnapshot"`
|
||||
SupportsSchemaEvents bool `json:"supportsSchemaEvents"`
|
||||
PreservesSourceTransactions bool `json:"preservesSourceTransactions"`
|
||||
RequiresCausalSnapshotReads bool `json:"requiresCausalSnapshotReads"`
|
||||
SnapshotSemantics string `json:"snapshotSemantics,omitempty"`
|
||||
DeliverySemantics string `json:"deliverySemantics,omitempty"`
|
||||
AcknowledgementSemantics string `json:"acknowledgementSemantics,omitempty"`
|
||||
}
|
||||
|
||||
type Stream interface {
|
||||
Next(context.Context) (Transaction, error)
|
||||
Acknowledge(context.Context, Position) error
|
||||
Close() error
|
||||
}
|
||||
|
||||
type Adapter interface {
|
||||
Name() string
|
||||
SourceTypes() []string
|
||||
Probe(context.Context, connection.ConnectionConfig) (Capability, error)
|
||||
BeginSnapshot(context.Context, Request) (Barrier, error)
|
||||
Open(context.Context, Request, Position) (Stream, error)
|
||||
}
|
||||
|
||||
var ErrAdapterNotRegistered = errors.New("CDC adapter is not registered in this build")
|
||||
|
||||
func ValidatePosition(position Position, adapterName string) error {
|
||||
if strings.TrimSpace(position.Adapter) == "" {
|
||||
return errors.New("CDC position adapter is required")
|
||||
}
|
||||
if !strings.EqualFold(strings.TrimSpace(position.Adapter), strings.TrimSpace(adapterName)) {
|
||||
return fmt.Errorf("CDC position adapter %q does not match %q", position.Adapter, adapterName)
|
||||
}
|
||||
if len(position.Opaque) == 0 || !json.Valid(position.Opaque) {
|
||||
return errors.New("CDC position payload must be valid JSON")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
383
internal/synccdc/mongodb_adapter.go
Normal file
383
internal/synccdc/mongodb_adapter.go
Normal file
@@ -0,0 +1,383 @@
|
||||
package synccdc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"GoNavi-Wails/internal/connection"
|
||||
"go.mongodb.org/mongo-driver/v2/bson"
|
||||
"go.mongodb.org/mongo-driver/v2/mongo"
|
||||
)
|
||||
|
||||
const (
|
||||
mongoSnapshotSemantics = "startAtOperationTime at-least-once barrier; complete handoff requires every snapshot read to be causally constrained at or after this operationTime, which this adapter cannot enforce by itself; without that constraint initial snapshot is unsupported, reads are not pinned to one cluster timestamp, and duplicate events are possible"
|
||||
mongoDeliverySemantics = "ordered per MongoDB change stream cursor with resumable at-least-once delivery; each returned transaction contains one change event; stream termination including filtered drop/invalidate returns a re-snapshot-required error"
|
||||
mongoAckSemantics = "local delivered-position validation only; MongoDB change streams have no server-side acknowledgement"
|
||||
)
|
||||
|
||||
var mongoRequiredSettings = []string{
|
||||
"MongoDB 4.2+ replica set or sharded cluster; standalone servers are unsupported",
|
||||
"database-scope changeStream/find privileges for Probe, plus find/changeStream access to every selected namespace",
|
||||
"majority read concern and oplog retention covering the full snapshot duration",
|
||||
"a resolved direct MongoDB endpoint; SSH, proxy, and HTTP tunnel routing must be established before CDC",
|
||||
"cluster-scope changeStream privilege when selected namespaces span multiple databases",
|
||||
"snapshot reader support for causal reads at or after the returned operationTime when initialSnapshot is enabled",
|
||||
}
|
||||
|
||||
type MongoDBAdapter struct {
|
||||
connector mongoConnector
|
||||
}
|
||||
|
||||
type mongoSnapshotToken struct {
|
||||
Version int `json:"version"`
|
||||
Strategy string `json:"strategy"`
|
||||
ScopeHash string `json:"scopeHash"`
|
||||
OperationTime mongoOperationTime `json:"operationTime"`
|
||||
Semantics string `json:"semantics"`
|
||||
}
|
||||
|
||||
func NewMongoDBAdapter() *MongoDBAdapter {
|
||||
return &MongoDBAdapter{connector: realMongoConnector{}}
|
||||
}
|
||||
|
||||
func newMongoDBAdapterWithConnector(connector mongoConnector) *MongoDBAdapter {
|
||||
return &MongoDBAdapter{connector: connector}
|
||||
}
|
||||
|
||||
func (a *MongoDBAdapter) Name() string {
|
||||
return mongoDBAdapterName
|
||||
}
|
||||
|
||||
func (a *MongoDBAdapter) SourceTypes() []string {
|
||||
return []string{"mongodb", "mongodb-v1"}
|
||||
}
|
||||
|
||||
func (a *MongoDBAdapter) Probe(ctx context.Context, config connection.ConnectionConfig) (Capability, error) {
|
||||
ctx = mongoContext(ctx)
|
||||
capability := mongoCapability(config.Type)
|
||||
if normalizeSourceType(config.Type) != "mongodb" {
|
||||
capability.Supported = false
|
||||
capability.Reason = fmt.Sprintf("source type %q is not handled by the MongoDB CDC adapter", strings.TrimSpace(config.Type))
|
||||
return capability, nil
|
||||
}
|
||||
if err := validateMongoNetworkRoute(config); err != nil {
|
||||
capability.Reason = err.Error()
|
||||
return capability, nil
|
||||
}
|
||||
database := mongoConfigDatabase(config)
|
||||
if database == "" {
|
||||
capability.Reason = "MongoDB CDC requires a source database for its privilege and change-stream probe"
|
||||
return capability, nil
|
||||
}
|
||||
if a == nil || a.connector == nil {
|
||||
return capability, errors.New("MongoDB CDC connector is not configured")
|
||||
}
|
||||
conn, err := a.connector.Connect(ctx, config)
|
||||
if err != nil {
|
||||
if ctxErr := contextError(ctx, err); ctxErr != nil {
|
||||
return capability, ctxErr
|
||||
}
|
||||
capability.Reason = classifyMongoProbeError(err)
|
||||
return capability, nil
|
||||
}
|
||||
defer disconnectMongoConnection(conn)
|
||||
|
||||
topology, err := conn.Inspect(ctx)
|
||||
if err != nil {
|
||||
if ctxErr := contextError(ctx, err); ctxErr != nil {
|
||||
return capability, ctxErr
|
||||
}
|
||||
capability.Reason = classifyMongoProbeError(err)
|
||||
return capability, nil
|
||||
}
|
||||
if reason := validateMongoTopology(topology); reason != "" {
|
||||
capability.Reason = reason
|
||||
return capability, nil
|
||||
}
|
||||
if err := conn.ProbeChangeStream(ctx, database, bson.Timestamp{}); err != nil {
|
||||
if ctxErr := contextError(ctx, err); ctxErr != nil {
|
||||
return capability, ctxErr
|
||||
}
|
||||
capability.Reason = classifyMongoProbeError(err)
|
||||
return capability, nil
|
||||
}
|
||||
capability.Ready = true
|
||||
capability.Reason = mongoSnapshotSemantics
|
||||
return capability, nil
|
||||
}
|
||||
|
||||
func (a *MongoDBAdapter) BeginSnapshot(ctx context.Context, request Request) (Barrier, error) {
|
||||
ctx = mongoContext(ctx)
|
||||
namespaces, scopeHash, err := validateMongoRequest(request)
|
||||
if err != nil {
|
||||
return Barrier{}, err
|
||||
}
|
||||
if a == nil || a.connector == nil {
|
||||
return Barrier{}, errors.New("MongoDB CDC connector is not configured")
|
||||
}
|
||||
conn, err := a.connector.Connect(ctx, request.Config)
|
||||
if err != nil {
|
||||
return Barrier{}, err
|
||||
}
|
||||
defer disconnectMongoConnection(conn)
|
||||
if err := ensureMongoTopology(ctx, conn); err != nil {
|
||||
return Barrier{}, err
|
||||
}
|
||||
operationTime, err := conn.SnapshotOperationTime(ctx, namespaces[0])
|
||||
if err != nil {
|
||||
return Barrier{}, err
|
||||
}
|
||||
position, err := mongoOperationTimePosition(scopeHash, operationTime)
|
||||
if err != nil {
|
||||
return Barrier{}, err
|
||||
}
|
||||
snapshotToken, err := json.Marshal(mongoSnapshotToken{
|
||||
Version: mongoPositionVersion,
|
||||
Strategy: "startAtOperationTime",
|
||||
ScopeHash: scopeHash,
|
||||
OperationTime: mongoOperationTime{
|
||||
Seconds: operationTime.T,
|
||||
Increment: operationTime.I,
|
||||
},
|
||||
Semantics: mongoSnapshotSemantics,
|
||||
})
|
||||
if err != nil {
|
||||
return Barrier{}, fmt.Errorf("encode MongoDB CDC snapshot token: %w", err)
|
||||
}
|
||||
return Barrier{Position: position, SnapshotToken: snapshotToken}, nil
|
||||
}
|
||||
|
||||
func (a *MongoDBAdapter) Open(ctx context.Context, request Request, position Position) (Stream, error) {
|
||||
ctx = mongoContext(ctx)
|
||||
namespaces, scopeHash, err := validateMongoRequest(request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
payload, resumeToken, operationTime, err := decodeMongoPosition(position)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if payload.ScopeHash != scopeHash {
|
||||
return nil, errors.New("MongoDB CDC position belongs to a different source or namespace selection")
|
||||
}
|
||||
if a == nil || a.connector == nil {
|
||||
return nil, errors.New("MongoDB CDC connector is not configured")
|
||||
}
|
||||
conn, err := a.connector.Connect(ctx, request.Config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ensureMongoTopology(ctx, conn); err != nil {
|
||||
disconnectMongoConnection(conn)
|
||||
return nil, err
|
||||
}
|
||||
cursor, err := conn.OpenChangeStream(ctx, namespaces, mongoWatchStart{
|
||||
ResumeToken: resumeToken,
|
||||
OperationTime: operationTime,
|
||||
})
|
||||
if err != nil {
|
||||
disconnectMongoConnection(conn)
|
||||
return nil, fmt.Errorf("open MongoDB change stream: %w", err)
|
||||
}
|
||||
return newMongoDBStream(conn, cursor, namespaces, scopeHash), nil
|
||||
}
|
||||
|
||||
func mongoCapability(sourceType string) Capability {
|
||||
normalizedSource := normalizeSourceType(sourceType)
|
||||
if normalizedSource == "" {
|
||||
normalizedSource = "mongodb"
|
||||
}
|
||||
return Capability{
|
||||
Adapter: mongoDBAdapterName,
|
||||
SourceType: normalizedSource,
|
||||
Supported: true,
|
||||
Ready: false,
|
||||
RequiredSettings: append([]string(nil), mongoRequiredSettings...),
|
||||
SupportsInitialSnapshot: true,
|
||||
SupportsSchemaEvents: false,
|
||||
PreservesSourceTransactions: false,
|
||||
RequiresCausalSnapshotReads: true,
|
||||
SnapshotSemantics: mongoSnapshotSemantics,
|
||||
DeliverySemantics: mongoDeliverySemantics,
|
||||
AcknowledgementSemantics: mongoAckSemantics,
|
||||
}
|
||||
}
|
||||
|
||||
func validateMongoRequest(request Request) ([]mongoNamespace, string, error) {
|
||||
if normalizeSourceType(request.Config.Type) != "mongodb" {
|
||||
return nil, "", fmt.Errorf("MongoDB CDC source type must be mongodb or mongodb-v1, got %q", strings.TrimSpace(request.Config.Type))
|
||||
}
|
||||
if err := validateMongoNetworkRoute(request.Config); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if len(request.Objects) == 0 {
|
||||
return nil, "", errors.New("MongoDB CDC requires at least one selected collection")
|
||||
}
|
||||
fallbackDatabase := firstNonEmpty(request.Database, request.Schema, mongoConfigDatabase(request.Config))
|
||||
seen := make(map[string]struct{}, len(request.Objects))
|
||||
namespaces := make([]mongoNamespace, 0, len(request.Objects))
|
||||
for _, object := range request.Objects {
|
||||
database := firstNonEmpty(object.Database, object.Schema, fallbackDatabase)
|
||||
collection := strings.TrimSpace(object.Name)
|
||||
if database == "" {
|
||||
return nil, "", fmt.Errorf("MongoDB CDC collection %q has no database namespace", collection)
|
||||
}
|
||||
if collection == "" {
|
||||
return nil, "", errors.New("MongoDB CDC collection name is required")
|
||||
}
|
||||
if strings.ContainsRune(database, '\x00') || strings.ContainsRune(collection, '\x00') {
|
||||
return nil, "", errors.New("MongoDB CDC namespaces must not contain NUL characters")
|
||||
}
|
||||
if strings.HasPrefix(collection, "system.") {
|
||||
return nil, "", fmt.Errorf("MongoDB CDC does not support system collection %s.%s", database, collection)
|
||||
}
|
||||
key := database + "\x00" + collection
|
||||
if _, exists := seen[key]; exists {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
namespaces = append(namespaces, mongoNamespace{Database: database, Collection: collection})
|
||||
}
|
||||
sort.Slice(namespaces, func(i, j int) bool {
|
||||
if namespaces[i].Database == namespaces[j].Database {
|
||||
return namespaces[i].Collection < namespaces[j].Collection
|
||||
}
|
||||
return namespaces[i].Database < namespaces[j].Database
|
||||
})
|
||||
return namespaces, mongoScopeHash(request.Config, namespaces), nil
|
||||
}
|
||||
|
||||
func mongoScopeHash(config connection.ConnectionConfig, namespaces []mongoNamespace) string {
|
||||
parts := []string{
|
||||
"v1",
|
||||
normalizeSourceType(config.Type),
|
||||
strings.TrimSpace(config.ID),
|
||||
strings.TrimSpace(config.ReplicaSet),
|
||||
mongoSafeEndpoint(config),
|
||||
}
|
||||
for _, namespace := range namespaces {
|
||||
parts = append(parts, namespace.Database+"."+namespace.Collection)
|
||||
}
|
||||
sum := sha256.Sum256([]byte(strings.Join(parts, "\x00")))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func mongoSafeEndpoint(config connection.ConnectionConfig) string {
|
||||
if rawURI := strings.TrimSpace(config.URI); rawURI != "" {
|
||||
if parsed, err := url.Parse(rawURI); err == nil {
|
||||
parsed.User = nil
|
||||
parsed.RawQuery = ""
|
||||
parsed.Fragment = ""
|
||||
return strings.ToLower(parsed.String())
|
||||
}
|
||||
}
|
||||
return strings.ToLower(strings.Join(normalizedMongoHosts(config), ",") + "/" + strings.TrimSpace(config.Database))
|
||||
}
|
||||
|
||||
func mongoConfigDatabase(config connection.ConnectionConfig) string {
|
||||
if database := strings.TrimSpace(config.Database); database != "" {
|
||||
return database
|
||||
}
|
||||
if rawURI := strings.TrimSpace(config.URI); rawURI != "" {
|
||||
if parsed, err := url.Parse(rawURI); err == nil {
|
||||
if database, err := url.PathUnescape(strings.Trim(parsed.EscapedPath(), "/")); err == nil {
|
||||
return strings.TrimSpace(database)
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func ensureMongoTopology(ctx context.Context, conn mongoConnection) error {
|
||||
topology, err := conn.Inspect(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if reason := validateMongoTopology(topology); reason != "" {
|
||||
return errors.New(reason)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateMongoTopology(topology mongoTopology) string {
|
||||
if strings.TrimSpace(topology.ReplicaSet) == "" && !topology.Sharded {
|
||||
return "MongoDB CDC is unavailable on standalone servers; configure a replica set or sharded cluster"
|
||||
}
|
||||
if topology.MaxWireVersion < mongoMinimumBarrierWireVersion {
|
||||
return fmt.Sprintf("MongoDB CDC requires MongoDB 4.2+ (maxWireVersion >= %d), server reported %d", mongoMinimumBarrierWireVersion, topology.MaxWireVersion)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func classifyMongoProbeError(err error) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
var commandError mongo.CommandError
|
||||
if errors.As(err, &commandError) {
|
||||
switch commandError.Code {
|
||||
case 13:
|
||||
return "MongoDB CDC probe was not authorized; grant changeStream and find privileges for the source database and selected collections"
|
||||
case 286:
|
||||
return "MongoDB change-stream history is no longer available; increase oplog retention and create a new snapshot barrier"
|
||||
case 40573, 40615:
|
||||
return "MongoDB change streams require a replica set or sharded cluster; standalone servers are unsupported"
|
||||
}
|
||||
}
|
||||
message := strings.ToLower(err.Error())
|
||||
switch {
|
||||
case strings.Contains(message, "not authorized"), strings.Contains(message, "unauthorized"):
|
||||
return "MongoDB CDC probe was not authorized; grant changeStream and find privileges for the source database and selected collections"
|
||||
case strings.Contains(message, "only supported on replica sets"), strings.Contains(message, "$changestream") && strings.Contains(message, "standalone"):
|
||||
return "MongoDB change streams require a replica set or sharded cluster; standalone servers are unsupported"
|
||||
case strings.Contains(message, "operationtime"):
|
||||
return "MongoDB did not provide an operationTime; enable majority read concern on a MongoDB 4.0+ replica set or sharded cluster"
|
||||
default:
|
||||
return "MongoDB CDC probe failed; verify the endpoint, TLS, credentials, replica-set settings, and source privileges"
|
||||
}
|
||||
}
|
||||
|
||||
func contextError(ctx context.Context, err error) error {
|
||||
if ctx != nil && ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func disconnectMongoConnection(conn mongoConnection) {
|
||||
if conn == nil {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
_ = conn.Disconnect(ctx)
|
||||
cancel()
|
||||
}
|
||||
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, value := range values {
|
||||
if trimmed := strings.TrimSpace(value); trimmed != "" {
|
||||
return trimmed
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func mongoContext(ctx context.Context) context.Context {
|
||||
if ctx == nil {
|
||||
return context.Background()
|
||||
}
|
||||
return ctx
|
||||
}
|
||||
|
||||
var _ Adapter = (*MongoDBAdapter)(nil)
|
||||
367
internal/synccdc/mongodb_adapter_test.go
Normal file
367
internal/synccdc/mongodb_adapter_test.go
Normal file
@@ -0,0 +1,367 @@
|
||||
package synccdc
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"GoNavi-Wails/internal/connection"
|
||||
"go.mongodb.org/mongo-driver/v2/bson"
|
||||
"go.mongodb.org/mongo-driver/v2/mongo"
|
||||
)
|
||||
|
||||
type fakeMongoConnector struct {
|
||||
connection *fakeMongoConnection
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeMongoConnector) Connect(context.Context, connection.ConnectionConfig) (mongoConnection, error) {
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
return f.connection, nil
|
||||
}
|
||||
|
||||
type fakeMongoConnection struct {
|
||||
mu sync.Mutex
|
||||
topology mongoTopology
|
||||
inspectErr error
|
||||
operationTime bson.Timestamp
|
||||
operationErr error
|
||||
probeErr error
|
||||
cursor mongoCursor
|
||||
openErr error
|
||||
openNamespaces []mongoNamespace
|
||||
openStart mongoWatchStart
|
||||
disconnects int
|
||||
}
|
||||
|
||||
func (f *fakeMongoConnection) Inspect(context.Context) (mongoTopology, error) {
|
||||
return f.topology, f.inspectErr
|
||||
}
|
||||
|
||||
func (f *fakeMongoConnection) SnapshotOperationTime(context.Context, mongoNamespace) (bson.Timestamp, error) {
|
||||
return f.operationTime, f.operationErr
|
||||
}
|
||||
|
||||
func (f *fakeMongoConnection) ProbeChangeStream(context.Context, string, bson.Timestamp) error {
|
||||
return f.probeErr
|
||||
}
|
||||
|
||||
func (f *fakeMongoConnection) OpenChangeStream(_ context.Context, namespaces []mongoNamespace, start mongoWatchStart) (mongoCursor, error) {
|
||||
f.mu.Lock()
|
||||
f.openNamespaces = append([]mongoNamespace(nil), namespaces...)
|
||||
f.openStart = start
|
||||
f.mu.Unlock()
|
||||
if f.openErr != nil {
|
||||
return nil, f.openErr
|
||||
}
|
||||
return f.cursor, nil
|
||||
}
|
||||
|
||||
func (f *fakeMongoConnection) Disconnect(context.Context) error {
|
||||
f.mu.Lock()
|
||||
f.disconnects++
|
||||
f.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeMongoCursor struct {
|
||||
mu sync.Mutex
|
||||
events []bson.Raw
|
||||
tokens []bson.Raw
|
||||
index int
|
||||
err error
|
||||
closed int
|
||||
nextStarted chan struct{}
|
||||
block bool
|
||||
}
|
||||
|
||||
func (f *fakeMongoCursor) Next(ctx context.Context) bool {
|
||||
if f.block {
|
||||
if f.nextStarted != nil {
|
||||
select {
|
||||
case f.nextStarted <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
<-ctx.Done()
|
||||
f.mu.Lock()
|
||||
f.err = ctx.Err()
|
||||
f.mu.Unlock()
|
||||
return false
|
||||
}
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.index < len(f.events)
|
||||
}
|
||||
|
||||
func (f *fakeMongoCursor) Decode(value any) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.index >= len(f.events) {
|
||||
return errors.New("no fake MongoDB event")
|
||||
}
|
||||
err := bson.Unmarshal(f.events[f.index], value)
|
||||
f.index++
|
||||
return err
|
||||
}
|
||||
|
||||
func (f *fakeMongoCursor) ResumeToken() bson.Raw {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.index == 0 || f.index > len(f.tokens) {
|
||||
return nil
|
||||
}
|
||||
return f.tokens[f.index-1]
|
||||
}
|
||||
|
||||
func (f *fakeMongoCursor) Err() error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.err
|
||||
}
|
||||
|
||||
func (f *fakeMongoCursor) Close(context.Context) error {
|
||||
f.mu.Lock()
|
||||
f.closed++
|
||||
f.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestMongoPositionRoundTripPreservesRawResumeToken(t *testing.T) {
|
||||
token := mustMongoRaw(t, bson.D{{Key: "_data", Value: "8268AABB"}, {Key: "nested", Value: bson.D{{Key: "n", Value: int64(42)}}}})
|
||||
position, err := mongoResumeTokenPosition("scope-1", token)
|
||||
if err != nil {
|
||||
t.Fatalf("encode position: %v", err)
|
||||
}
|
||||
payload, decodedToken, operationTime, err := decodeMongoPosition(position)
|
||||
if err != nil {
|
||||
t.Fatalf("decode position: %v", err)
|
||||
}
|
||||
if payload.ScopeHash != "scope-1" || operationTime != nil {
|
||||
t.Fatalf("unexpected decoded payload: %+v, operationTime=%v", payload, operationTime)
|
||||
}
|
||||
if !bytes.Equal(decodedToken, token) {
|
||||
t.Fatalf("resume token changed: got %v want %v", decodedToken, token)
|
||||
}
|
||||
|
||||
position.Opaque = json.RawMessage(`{"version":1,"scopeHash":"scope-1","resumeFormat":"bson-base64-v1","resumeTokenBson":"not@base64"}`)
|
||||
if _, _, _, err := decodeMongoPosition(position); err == nil {
|
||||
t.Fatal("malformed base64 resume token must be rejected")
|
||||
}
|
||||
validPosition, err := mongoResumeTokenPosition("scope-1", token)
|
||||
if err != nil {
|
||||
t.Fatalf("re-encode position: %v", err)
|
||||
}
|
||||
validPosition.Opaque = append(validPosition.Opaque, []byte(` {}`)...)
|
||||
if _, _, _, err := decodeMongoPosition(validPosition); err == nil {
|
||||
t.Fatal("trailing JSON must be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMongoBeginSnapshotUsesOperationTimeBarrier(t *testing.T) {
|
||||
fakeConnection := &fakeMongoConnection{
|
||||
topology: mongoTopology{ReplicaSet: "rs0", MaxWireVersion: 17},
|
||||
operationTime: bson.Timestamp{T: 1_725_000_000, I: 7},
|
||||
}
|
||||
adapter := newMongoDBAdapterWithConnector(&fakeMongoConnector{connection: fakeConnection})
|
||||
request := mongoTestRequest()
|
||||
barrier, err := adapter.BeginSnapshot(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("begin snapshot: %v", err)
|
||||
}
|
||||
payload, token, operationTime, err := decodeMongoPosition(barrier.Position)
|
||||
if err != nil {
|
||||
t.Fatalf("decode barrier position: %v", err)
|
||||
}
|
||||
if len(token) != 0 || operationTime == nil || *operationTime != fakeConnection.operationTime {
|
||||
t.Fatalf("unexpected barrier: payload=%+v token=%v operationTime=%v", payload, token, operationTime)
|
||||
}
|
||||
var snapshot mongoSnapshotToken
|
||||
if err := json.Unmarshal(barrier.SnapshotToken, &snapshot); err != nil {
|
||||
t.Fatalf("decode snapshot token: %v", err)
|
||||
}
|
||||
if snapshot.Strategy != "startAtOperationTime" || !strings.Contains(snapshot.Semantics, "at-least-once") {
|
||||
t.Fatalf("snapshot semantics not explicit: %+v", snapshot)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMongoOpenUsesResumeTokenAndSelectedNamespaces(t *testing.T) {
|
||||
resumeToken := mustMongoRaw(t, bson.D{{Key: "_data", Value: "resume-1"}})
|
||||
fakeConnection := &fakeMongoConnection{
|
||||
topology: mongoTopology{ReplicaSet: "rs0", MaxWireVersion: 17},
|
||||
cursor: &fakeMongoCursor{},
|
||||
}
|
||||
adapter := newMongoDBAdapterWithConnector(&fakeMongoConnector{connection: fakeConnection})
|
||||
request := mongoTestRequest()
|
||||
namespaces, scopeHash, err := validateMongoRequest(request)
|
||||
if err != nil {
|
||||
t.Fatalf("validate request: %v", err)
|
||||
}
|
||||
position, err := mongoResumeTokenPosition(scopeHash, resumeToken)
|
||||
if err != nil {
|
||||
t.Fatalf("encode resume position: %v", err)
|
||||
}
|
||||
stream, err := adapter.Open(context.Background(), request, position)
|
||||
if err != nil {
|
||||
t.Fatalf("open stream: %v", err)
|
||||
}
|
||||
defer stream.Close()
|
||||
if len(fakeConnection.openStart.ResumeToken) == 0 || fakeConnection.openStart.OperationTime != nil {
|
||||
t.Fatalf("resume start was not used: %+v", fakeConnection.openStart)
|
||||
}
|
||||
if len(fakeConnection.openNamespaces) != len(namespaces) {
|
||||
t.Fatalf("namespaces = %+v, want %+v", fakeConnection.openNamespaces, namespaces)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMongoProbeClassifiesTopologyPrivilegeAndReadyState(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
connection *fakeMongoConnection
|
||||
ready bool
|
||||
reason string
|
||||
}{
|
||||
{
|
||||
name: "standalone",
|
||||
connection: &fakeMongoConnection{topology: mongoTopology{MaxWireVersion: 17}},
|
||||
reason: "standalone",
|
||||
},
|
||||
{
|
||||
name: "not authorized",
|
||||
connection: &fakeMongoConnection{
|
||||
topology: mongoTopology{ReplicaSet: "rs0", MaxWireVersion: 17},
|
||||
operationTime: bson.Timestamp{T: 100, I: 1},
|
||||
probeErr: mongo.CommandError{Code: 13, Message: "not authorized"},
|
||||
},
|
||||
reason: "changeStream and find",
|
||||
},
|
||||
{
|
||||
name: "ready",
|
||||
connection: &fakeMongoConnection{
|
||||
topology: mongoTopology{ReplicaSet: "rs0", MaxWireVersion: 17},
|
||||
operationTime: bson.Timestamp{T: 100, I: 1},
|
||||
},
|
||||
ready: true,
|
||||
reason: "at-least-once",
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
adapter := newMongoDBAdapterWithConnector(&fakeMongoConnector{connection: test.connection})
|
||||
capability, err := adapter.Probe(context.Background(), mongoTestRequest().Config)
|
||||
if err != nil {
|
||||
t.Fatalf("probe: %v", err)
|
||||
}
|
||||
if capability.Ready != test.ready {
|
||||
t.Fatalf("ready=%v, want %v; reason=%q", capability.Ready, test.ready, capability.Reason)
|
||||
}
|
||||
if !strings.Contains(capability.Reason, test.reason) {
|
||||
t.Fatalf("reason %q does not contain %q", capability.Reason, test.reason)
|
||||
}
|
||||
if capability.PreservesSourceTransactions {
|
||||
t.Fatal("adapter must not claim transaction grouping")
|
||||
}
|
||||
if !capability.RequiresCausalSnapshotReads || !strings.Contains(capability.SnapshotSemantics, "cannot enforce") {
|
||||
t.Fatalf("snapshot consistency condition is not explicit: %+v", capability)
|
||||
}
|
||||
if !strings.Contains(capability.AcknowledgementSemantics, "no server-side") {
|
||||
t.Fatalf("acknowledgement semantics are misleading: %q", capability.AcknowledgementSemantics)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMongoRequestRejectsEmptyObjectsAndUnresolvedRoutes(t *testing.T) {
|
||||
request := mongoTestRequest()
|
||||
request.Objects = nil
|
||||
if _, _, err := validateMongoRequest(request); err == nil {
|
||||
t.Fatal("empty object selection must be rejected")
|
||||
}
|
||||
request = mongoTestRequest()
|
||||
request.Config.UseSSH = true
|
||||
if _, _, err := validateMongoRequest(request); err == nil || !strings.Contains(err.Error(), "resolved direct endpoint") {
|
||||
t.Fatalf("unresolved SSH route error = %v", err)
|
||||
}
|
||||
request = mongoTestRequest()
|
||||
request.Config.UseProxy = true
|
||||
if _, _, err := validateMongoRequest(request); err == nil || !strings.Contains(err.Error(), "resolved direct endpoint") {
|
||||
t.Fatalf("unresolved proxy route error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMongoConnectionURIAndScopeNeverPersistCredentials(t *testing.T) {
|
||||
config := connection.ConnectionConfig{
|
||||
Type: "mongodb",
|
||||
URI: "mongodb://secret-user:secret-password@mongo.example:27017/app?replicaSet=rs0",
|
||||
Database: "app",
|
||||
ReplicaSet: "rs0",
|
||||
}
|
||||
uri, err := mongoConnectionURI(config)
|
||||
if err != nil {
|
||||
t.Fatalf("build URI: %v", err)
|
||||
}
|
||||
if !strings.Contains(uri, "secret-password") {
|
||||
t.Fatal("driver URI unexpectedly discarded runtime credentials")
|
||||
}
|
||||
safeEndpoint := mongoSafeEndpoint(config)
|
||||
if strings.Contains(safeEndpoint, "secret-user") || strings.Contains(safeEndpoint, "secret-password") {
|
||||
t.Fatalf("safe endpoint leaked credentials: %q", safeEndpoint)
|
||||
}
|
||||
request := mongoTestRequest()
|
||||
request.Config = config
|
||||
_, scopeHash, err := validateMongoRequest(request)
|
||||
if err != nil {
|
||||
t.Fatalf("validate request: %v", err)
|
||||
}
|
||||
if strings.Contains(scopeHash, "secret") || len(scopeHash) != 64 {
|
||||
t.Fatalf("scope hash is not an opaque SHA-256 value: %q", scopeHash)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMongoAuthAttemptsPreferPrimaryThenReplicaCredentials(t *testing.T) {
|
||||
config := connection.ConnectionConfig{
|
||||
User: "primary-user",
|
||||
Password: "primary-password",
|
||||
MongoReplicaUser: "cdc-user",
|
||||
MongoReplicaPassword: "cdc-password",
|
||||
}
|
||||
attempts := mongoAuthAttempts(config)
|
||||
if len(attempts) != 2 || attempts[0].User != "primary-user" || attempts[1].User != "cdc-user" {
|
||||
t.Fatalf("unexpected auth attempts: %+v", attempts)
|
||||
}
|
||||
if attempts[1].MongoReplicaPassword != "" {
|
||||
t.Fatal("replica attempt must not recursively retain alternate credentials")
|
||||
}
|
||||
}
|
||||
|
||||
func mongoTestRequest() Request {
|
||||
return Request{
|
||||
Config: connection.ConnectionConfig{
|
||||
ID: "source-1",
|
||||
Type: "mongodb-v1",
|
||||
Host: "mongo.internal",
|
||||
Port: 27017,
|
||||
Database: "app",
|
||||
ReplicaSet: "rs0",
|
||||
},
|
||||
Objects: []ObjectRef{
|
||||
{Database: "app", Name: "orders"},
|
||||
{Database: "app", Name: "customers"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func mustMongoRaw(t *testing.T, value any) bson.Raw {
|
||||
t.Helper()
|
||||
raw, err := bson.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal BSON: %v", err)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
452
internal/synccdc/mongodb_driver.go
Normal file
452
internal/synccdc/mongodb_driver.go
Normal file
@@ -0,0 +1,452 @@
|
||||
package synccdc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"GoNavi-Wails/internal/connection"
|
||||
"GoNavi-Wails/internal/tlsconfig"
|
||||
"go.mongodb.org/mongo-driver/v2/bson"
|
||||
"go.mongodb.org/mongo-driver/v2/mongo"
|
||||
"go.mongodb.org/mongo-driver/v2/mongo/options"
|
||||
"go.mongodb.org/mongo-driver/v2/mongo/readconcern"
|
||||
"go.mongodb.org/mongo-driver/v2/mongo/readpref"
|
||||
)
|
||||
|
||||
const (
|
||||
mongoDefaultPort = 27017
|
||||
mongoDefaultConnectTimeout = 30 * time.Second
|
||||
mongoMinimumBarrierWireVersion int32 = 8 // mongo-driver/v2 supports MongoDB 4.2+.
|
||||
)
|
||||
|
||||
type mongoNamespace struct {
|
||||
Database string
|
||||
Collection string
|
||||
}
|
||||
|
||||
type mongoTopology struct {
|
||||
ReplicaSet string
|
||||
Sharded bool
|
||||
MaxWireVersion int32
|
||||
}
|
||||
|
||||
type mongoWatchStart struct {
|
||||
ResumeToken bson.Raw
|
||||
OperationTime *bson.Timestamp
|
||||
}
|
||||
|
||||
type mongoCursor interface {
|
||||
Next(context.Context) bool
|
||||
Decode(any) error
|
||||
ResumeToken() bson.Raw
|
||||
Err() error
|
||||
Close(context.Context) error
|
||||
}
|
||||
|
||||
type mongoConnection interface {
|
||||
Inspect(context.Context) (mongoTopology, error)
|
||||
SnapshotOperationTime(context.Context, mongoNamespace) (bson.Timestamp, error)
|
||||
ProbeChangeStream(context.Context, string, bson.Timestamp) error
|
||||
OpenChangeStream(context.Context, []mongoNamespace, mongoWatchStart) (mongoCursor, error)
|
||||
Disconnect(context.Context) error
|
||||
}
|
||||
|
||||
type mongoConnector interface {
|
||||
Connect(context.Context, connection.ConnectionConfig) (mongoConnection, error)
|
||||
}
|
||||
|
||||
type realMongoConnector struct{}
|
||||
|
||||
type realMongoConnection struct {
|
||||
client *mongo.Client
|
||||
}
|
||||
|
||||
type mongoHelloResponse struct {
|
||||
SetName string `bson:"setName"`
|
||||
Message string `bson:"msg"`
|
||||
MaxWireVersion int32 `bson:"maxWireVersion"`
|
||||
}
|
||||
|
||||
func (realMongoConnector) Connect(ctx context.Context, config connection.ConnectionConfig) (mongoConnection, error) {
|
||||
var lastErr error
|
||||
for _, attempt := range mongoAuthAttempts(config) {
|
||||
clientOptions, err := mongoClientOptions(attempt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client, err := mongo.Connect(clientOptions)
|
||||
if err != nil {
|
||||
lastErr = fmt.Errorf("connect to MongoDB for CDC: %w", err)
|
||||
continue
|
||||
}
|
||||
if err := client.Ping(ctx, readpref.Primary()); err != nil {
|
||||
disconnectCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
_ = client.Disconnect(disconnectCtx)
|
||||
cancel()
|
||||
lastErr = fmt.Errorf("ping MongoDB for CDC: %w", err)
|
||||
if ctxErr := contextError(ctx, err); ctxErr != nil {
|
||||
return nil, ctxErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return &realMongoConnection{client: client}, nil
|
||||
}
|
||||
if lastErr == nil {
|
||||
lastErr = errors.New("MongoDB CDC has no usable authentication configuration")
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
func (c *realMongoConnection) Inspect(ctx context.Context) (mongoTopology, error) {
|
||||
if c == nil || c.client == nil {
|
||||
return mongoTopology{}, errors.New("MongoDB CDC connection is closed")
|
||||
}
|
||||
var result mongoHelloResponse
|
||||
if err := c.client.Database("admin").RunCommand(ctx, bson.D{{Key: "hello", Value: 1}}).Decode(&result); err != nil {
|
||||
var commandError mongo.CommandError
|
||||
if !errors.As(err, &commandError) || commandError.Code != 59 {
|
||||
return mongoTopology{}, fmt.Errorf("inspect MongoDB topology: %w", err)
|
||||
}
|
||||
if err := c.client.Database("admin").RunCommand(ctx, bson.D{{Key: "isMaster", Value: 1}}).Decode(&result); err != nil {
|
||||
return mongoTopology{}, fmt.Errorf("inspect MongoDB topology: %w", err)
|
||||
}
|
||||
}
|
||||
return mongoTopology{
|
||||
ReplicaSet: strings.TrimSpace(result.SetName),
|
||||
Sharded: strings.EqualFold(strings.TrimSpace(result.Message), "isdbgrid"),
|
||||
MaxWireVersion: result.MaxWireVersion,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *realMongoConnection) SnapshotOperationTime(ctx context.Context, namespace mongoNamespace) (bson.Timestamp, error) {
|
||||
if c == nil || c.client == nil {
|
||||
return bson.Timestamp{}, errors.New("MongoDB CDC connection is closed")
|
||||
}
|
||||
var operationTime bson.Timestamp
|
||||
err := c.client.UseSession(ctx, func(sessionContext context.Context) error {
|
||||
result := c.client.Database(namespace.Database).Collection(namespace.Collection).FindOne(
|
||||
sessionContext,
|
||||
bson.D{},
|
||||
options.FindOne().SetProjection(bson.D{{Key: "_id", Value: 1}}),
|
||||
)
|
||||
if err := result.Err(); err != nil && !errors.Is(err, mongo.ErrNoDocuments) {
|
||||
return fmt.Errorf("establish majority snapshot barrier for %s.%s: %w", namespace.Database, namespace.Collection, err)
|
||||
}
|
||||
session := mongo.SessionFromContext(sessionContext)
|
||||
if session == nil || session.OperationTime() == nil {
|
||||
return errors.New("MongoDB did not return an operationTime for the snapshot barrier")
|
||||
}
|
||||
operationTime = *session.OperationTime()
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return bson.Timestamp{}, err
|
||||
}
|
||||
if operationTime.IsZero() {
|
||||
return bson.Timestamp{}, errors.New("MongoDB returned an empty operationTime for the snapshot barrier")
|
||||
}
|
||||
return operationTime, nil
|
||||
}
|
||||
|
||||
func (c *realMongoConnection) ProbeChangeStream(ctx context.Context, database string, operationTime bson.Timestamp) error {
|
||||
if c == nil || c.client == nil {
|
||||
return errors.New("MongoDB CDC connection is closed")
|
||||
}
|
||||
pipeline := mongo.Pipeline{bson.D{{Key: "$match", Value: bson.D{
|
||||
{Key: "operationType", Value: "__gonavi_cdc_probe__"},
|
||||
}}}}
|
||||
streamOptions := options.ChangeStream().
|
||||
SetFullDocument(options.UpdateLookup).
|
||||
SetMaxAwaitTime(250 * time.Millisecond)
|
||||
if !operationTime.IsZero() {
|
||||
streamOptions.SetStartAtOperationTime(&operationTime)
|
||||
}
|
||||
stream, err := c.client.Database(database).Watch(ctx, pipeline, streamOptions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return stream.Close(ctx)
|
||||
}
|
||||
|
||||
func (c *realMongoConnection) OpenChangeStream(ctx context.Context, namespaces []mongoNamespace, start mongoWatchStart) (mongoCursor, error) {
|
||||
if c == nil || c.client == nil {
|
||||
return nil, errors.New("MongoDB CDC connection is closed")
|
||||
}
|
||||
pipeline := mongoNamespacePipeline(namespaces)
|
||||
streamOptions := options.ChangeStream().
|
||||
SetFullDocument(options.UpdateLookup).
|
||||
SetMaxAwaitTime(time.Second)
|
||||
if len(start.ResumeToken) > 0 {
|
||||
streamOptions.SetResumeAfter(start.ResumeToken)
|
||||
} else if start.OperationTime != nil {
|
||||
streamOptions.SetStartAtOperationTime(start.OperationTime)
|
||||
} else {
|
||||
return nil, errors.New("MongoDB CDC change stream start position is required")
|
||||
}
|
||||
|
||||
if len(namespaces) == 1 {
|
||||
namespace := namespaces[0]
|
||||
return c.client.Database(namespace.Database).Collection(namespace.Collection).Watch(ctx, pipeline, streamOptions)
|
||||
}
|
||||
if database, ok := singleMongoDatabase(namespaces); ok {
|
||||
return c.client.Database(database).Watch(ctx, pipeline, streamOptions)
|
||||
}
|
||||
return c.client.Watch(ctx, pipeline, streamOptions)
|
||||
}
|
||||
|
||||
func (c *realMongoConnection) Disconnect(ctx context.Context) error {
|
||||
if c == nil || c.client == nil {
|
||||
return nil
|
||||
}
|
||||
client := c.client
|
||||
c.client = nil
|
||||
return client.Disconnect(ctx)
|
||||
}
|
||||
|
||||
func mongoNamespacePipeline(namespaces []mongoNamespace) mongo.Pipeline {
|
||||
operationFilter := bson.D{{Key: "$in", Value: bson.A{"insert", "replace", "update", "delete"}}}
|
||||
match := bson.D{{Key: "operationType", Value: operationFilter}}
|
||||
if len(namespaces) == 1 {
|
||||
match = append(match,
|
||||
bson.E{Key: "ns.db", Value: namespaces[0].Database},
|
||||
bson.E{Key: "ns.coll", Value: namespaces[0].Collection},
|
||||
)
|
||||
} else if database, ok := singleMongoDatabase(namespaces); ok {
|
||||
collections := make(bson.A, 0, len(namespaces))
|
||||
for _, namespace := range namespaces {
|
||||
collections = append(collections, namespace.Collection)
|
||||
}
|
||||
match = append(match,
|
||||
bson.E{Key: "ns.db", Value: database},
|
||||
bson.E{Key: "ns.coll", Value: bson.D{{Key: "$in", Value: collections}}},
|
||||
)
|
||||
} else {
|
||||
alternatives := make(bson.A, 0, len(namespaces))
|
||||
for _, namespace := range namespaces {
|
||||
alternatives = append(alternatives, bson.D{
|
||||
{Key: "ns.db", Value: namespace.Database},
|
||||
{Key: "ns.coll", Value: namespace.Collection},
|
||||
})
|
||||
}
|
||||
match = append(match, bson.E{Key: "$or", Value: alternatives})
|
||||
}
|
||||
return mongo.Pipeline{bson.D{{Key: "$match", Value: match}}}
|
||||
}
|
||||
|
||||
func singleMongoDatabase(namespaces []mongoNamespace) (string, bool) {
|
||||
if len(namespaces) == 0 {
|
||||
return "", false
|
||||
}
|
||||
database := namespaces[0].Database
|
||||
for _, namespace := range namespaces[1:] {
|
||||
if namespace.Database != database {
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
return database, true
|
||||
}
|
||||
|
||||
func mongoClientOptions(config connection.ConnectionConfig) (*options.ClientOptions, error) {
|
||||
if err := validateMongoNetworkRoute(config); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
uri, err := mongoConnectionURI(config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clientOptions := options.Client().ApplyURI(uri).
|
||||
SetReadConcern(readconcern.Majority()).
|
||||
SetReadPreference(readpref.Primary())
|
||||
timeout := time.Duration(config.Timeout) * time.Second
|
||||
if timeout <= 0 {
|
||||
timeout = mongoDefaultConnectTimeout
|
||||
}
|
||||
clientOptions.SetConnectTimeout(timeout).SetServerSelectionTimeout(timeout)
|
||||
|
||||
username := strings.TrimSpace(config.User)
|
||||
password := config.Password
|
||||
mechanism := strings.TrimSpace(config.MongoAuthMechanism)
|
||||
if username != "" && !strings.EqualFold(mechanism, "NONE") {
|
||||
authSource := strings.TrimSpace(config.AuthSource)
|
||||
if authSource == "" {
|
||||
authSource = "admin"
|
||||
}
|
||||
clientOptions.SetAuth(options.Credential{
|
||||
AuthMechanism: mechanism,
|
||||
AuthSource: authSource,
|
||||
Username: username,
|
||||
Password: password,
|
||||
PasswordSet: true,
|
||||
})
|
||||
}
|
||||
if replicaSet := strings.TrimSpace(config.ReplicaSet); replicaSet != "" {
|
||||
clientOptions.SetReplicaSet(replicaSet)
|
||||
}
|
||||
tlsConfig, err := mongoCDCClientTLSConfig(config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tlsConfig != nil {
|
||||
clientOptions.SetTLSConfig(tlsConfig)
|
||||
}
|
||||
return clientOptions, nil
|
||||
}
|
||||
|
||||
func mongoAuthAttempts(config connection.ConnectionConfig) []connection.ConnectionConfig {
|
||||
replicaUser := strings.TrimSpace(config.MongoReplicaUser)
|
||||
primaryUser := strings.TrimSpace(config.User)
|
||||
if replicaUser == "" || (replicaUser == primaryUser && config.MongoReplicaPassword == config.Password) {
|
||||
return []connection.ConnectionConfig{config}
|
||||
}
|
||||
replicaAttempt := config
|
||||
replicaAttempt.User = replicaUser
|
||||
replicaAttempt.Password = config.MongoReplicaPassword
|
||||
replicaAttempt.MongoReplicaUser = ""
|
||||
replicaAttempt.MongoReplicaPassword = ""
|
||||
if primaryUser == "" {
|
||||
return []connection.ConnectionConfig{replicaAttempt}
|
||||
}
|
||||
return []connection.ConnectionConfig{config, replicaAttempt}
|
||||
}
|
||||
|
||||
func validateMongoNetworkRoute(config connection.ConnectionConfig) error {
|
||||
if config.UseSSH {
|
||||
return errors.New("MongoDB CDC requires a resolved direct endpoint; unresolved SSH tunnelling is not supported")
|
||||
}
|
||||
if config.UseProxy {
|
||||
return errors.New("MongoDB CDC requires a resolved direct endpoint; unresolved proxy routing is not supported")
|
||||
}
|
||||
if config.UseHTTPTunnel {
|
||||
return errors.New("MongoDB CDC requires a resolved direct endpoint; unresolved HTTP tunnelling is not supported")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func mongoConnectionURI(config connection.ConnectionConfig) (string, error) {
|
||||
rawURI := strings.TrimSpace(config.URI)
|
||||
if rawURI != "" {
|
||||
parsed, err := url.Parse(rawURI)
|
||||
if err != nil {
|
||||
return "", errors.New("MongoDB CDC URI is invalid")
|
||||
}
|
||||
if parsed.Scheme != "mongodb" && parsed.Scheme != "mongodb+srv" {
|
||||
return "", fmt.Errorf("MongoDB CDC URI scheme must be mongodb or mongodb+srv, got %q", parsed.Scheme)
|
||||
}
|
||||
if strings.TrimSpace(parsed.Host) == "" {
|
||||
return "", errors.New("MongoDB CDC URI host is required")
|
||||
}
|
||||
if err := mergeMongoConnectionParams(parsed.Query(), config.ConnectionParams, parsed); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return parsed.String(), nil
|
||||
}
|
||||
|
||||
hosts := normalizedMongoHosts(config)
|
||||
if len(hosts) == 0 {
|
||||
return "", errors.New("MongoDB CDC host is required")
|
||||
}
|
||||
if config.MongoSRV && len(hosts) != 1 {
|
||||
return "", errors.New("MongoDB SRV CDC configuration requires exactly one host")
|
||||
}
|
||||
scheme := "mongodb"
|
||||
if config.MongoSRV {
|
||||
scheme = "mongodb+srv"
|
||||
}
|
||||
database := strings.TrimSpace(config.Database)
|
||||
path := "/" + database
|
||||
parsed := &url.URL{Scheme: scheme, Host: strings.Join(hosts, ","), Path: path}
|
||||
params := parsed.Query()
|
||||
if replicaSet := strings.TrimSpace(config.ReplicaSet); replicaSet != "" {
|
||||
params.Set("replicaSet", replicaSet)
|
||||
}
|
||||
if authSource := strings.TrimSpace(config.AuthSource); authSource != "" {
|
||||
params.Set("authSource", authSource)
|
||||
}
|
||||
if mechanism := strings.TrimSpace(config.MongoAuthMechanism); mechanism != "" && !strings.EqualFold(mechanism, "NONE") {
|
||||
params.Set("authMechanism", mechanism)
|
||||
}
|
||||
if err := mergeMongoConnectionParams(params, config.ConnectionParams, parsed); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return parsed.String(), nil
|
||||
}
|
||||
|
||||
func mergeMongoConnectionParams(params url.Values, raw string, target *url.URL) error {
|
||||
raw = strings.TrimPrefix(strings.TrimSpace(raw), "?")
|
||||
raw = strings.ReplaceAll(raw, ";", "&")
|
||||
if raw != "" {
|
||||
parsed, err := url.ParseQuery(raw)
|
||||
if err != nil {
|
||||
return errors.New("MongoDB CDC connection parameters are invalid")
|
||||
}
|
||||
for key, values := range parsed {
|
||||
params.Del(key)
|
||||
for _, value := range values {
|
||||
params.Add(key, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
target.RawQuery = params.Encode()
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizedMongoHosts(config connection.ConnectionConfig) []string {
|
||||
rawHosts := append([]string(nil), config.Hosts...)
|
||||
if len(rawHosts) == 0 && strings.TrimSpace(config.Host) != "" {
|
||||
rawHosts = []string{config.Host}
|
||||
}
|
||||
seen := make(map[string]struct{}, len(rawHosts))
|
||||
hosts := make([]string, 0, len(rawHosts))
|
||||
for _, rawHost := range rawHosts {
|
||||
host := strings.TrimSpace(rawHost)
|
||||
if host == "" {
|
||||
continue
|
||||
}
|
||||
if config.MongoSRV {
|
||||
host = strings.TrimSuffix(host, ".")
|
||||
} else if _, _, err := net.SplitHostPort(host); err != nil {
|
||||
port := config.Port
|
||||
if port <= 0 {
|
||||
port = mongoDefaultPort
|
||||
}
|
||||
host = net.JoinHostPort(strings.Trim(host, "[]"), strconv.Itoa(port))
|
||||
}
|
||||
if _, exists := seen[host]; exists {
|
||||
continue
|
||||
}
|
||||
seen[host] = struct{}{}
|
||||
hosts = append(hosts, host)
|
||||
}
|
||||
sort.Strings(hosts)
|
||||
return hosts
|
||||
}
|
||||
|
||||
func mongoCDCClientTLSConfig(config connection.ConnectionConfig) (*tls.Config, error) {
|
||||
if !config.UseSSL {
|
||||
return nil, nil
|
||||
}
|
||||
mode := strings.ToLower(strings.TrimSpace(config.SSLMode))
|
||||
insecure := mode == "" || mode == "preferred" || mode == "prefer" || mode == "skip-verify" || mode == "skipverify" || mode == "insecure"
|
||||
if mode == "disable" || mode == "disabled" || mode == "off" || mode == "false" || mode == "none" {
|
||||
return nil, nil
|
||||
}
|
||||
tlsConfig, err := tlsconfig.BuildClientConfig(tlsconfig.ClientConfigOptions{
|
||||
Enabled: true,
|
||||
InsecureSkipVerify: insecure,
|
||||
CAPath: config.SSLCAPath,
|
||||
CertPath: config.SSLCertPath,
|
||||
KeyPath: config.SSLKeyPath,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("configure MongoDB CDC TLS: %w", err)
|
||||
}
|
||||
return tlsConfig, nil
|
||||
}
|
||||
136
internal/synccdc/mongodb_position.go
Normal file
136
internal/synccdc/mongodb_position.go
Normal file
@@ -0,0 +1,136 @@
|
||||
package synccdc
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"go.mongodb.org/mongo-driver/v2/bson"
|
||||
)
|
||||
|
||||
const (
|
||||
mongoDBAdapterName = "mongodb-change-stream"
|
||||
mongoPositionVersion = 1
|
||||
mongoResumeTokenFormat = "bson-base64-v1"
|
||||
mongoMaxPositionBytes = 2 << 20
|
||||
mongoMaxResumeTokenBytes = 1 << 20
|
||||
)
|
||||
|
||||
type mongoOperationTime struct {
|
||||
Seconds uint32 `json:"seconds"`
|
||||
Increment uint32 `json:"increment"`
|
||||
}
|
||||
|
||||
type mongoPositionPayload struct {
|
||||
Version int `json:"version"`
|
||||
ScopeHash string `json:"scopeHash"`
|
||||
ResumeTokenBSON string `json:"resumeTokenBson,omitempty"`
|
||||
ResumeFormat string `json:"resumeFormat,omitempty"`
|
||||
OperationTime *mongoOperationTime `json:"operationTime,omitempty"`
|
||||
}
|
||||
|
||||
func mongoOperationTimePosition(scopeHash string, timestamp bson.Timestamp) (Position, error) {
|
||||
if timestamp.IsZero() {
|
||||
return Position{}, errors.New("MongoDB CDC operation time is empty")
|
||||
}
|
||||
return marshalMongoPosition(mongoPositionPayload{
|
||||
Version: mongoPositionVersion,
|
||||
ScopeHash: strings.TrimSpace(scopeHash),
|
||||
OperationTime: &mongoOperationTime{
|
||||
Seconds: timestamp.T,
|
||||
Increment: timestamp.I,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func mongoResumeTokenPosition(scopeHash string, token bson.Raw) (Position, error) {
|
||||
if len(token) == 0 {
|
||||
return Position{}, errors.New("MongoDB CDC resume token is empty")
|
||||
}
|
||||
if err := token.Validate(); err != nil {
|
||||
return Position{}, fmt.Errorf("MongoDB CDC resume token is invalid BSON: %w", err)
|
||||
}
|
||||
return marshalMongoPosition(mongoPositionPayload{
|
||||
Version: mongoPositionVersion,
|
||||
ScopeHash: strings.TrimSpace(scopeHash),
|
||||
ResumeTokenBSON: base64.RawURLEncoding.EncodeToString(token),
|
||||
ResumeFormat: mongoResumeTokenFormat,
|
||||
})
|
||||
}
|
||||
|
||||
func marshalMongoPosition(payload mongoPositionPayload) (Position, error) {
|
||||
if strings.TrimSpace(payload.ScopeHash) == "" {
|
||||
return Position{}, errors.New("MongoDB CDC position scope is required")
|
||||
}
|
||||
opaque, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return Position{}, fmt.Errorf("encode MongoDB CDC position: %w", err)
|
||||
}
|
||||
return Position{Adapter: mongoDBAdapterName, Opaque: opaque}, nil
|
||||
}
|
||||
|
||||
func decodeMongoPosition(position Position) (mongoPositionPayload, bson.Raw, *bson.Timestamp, error) {
|
||||
if err := ValidatePosition(position, mongoDBAdapterName); err != nil {
|
||||
return mongoPositionPayload{}, nil, nil, err
|
||||
}
|
||||
if len(position.Opaque) > mongoMaxPositionBytes {
|
||||
return mongoPositionPayload{}, nil, nil, errors.New("MongoDB CDC position exceeds the supported size")
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(position.Opaque))
|
||||
decoder.DisallowUnknownFields()
|
||||
var payload mongoPositionPayload
|
||||
if err := decoder.Decode(&payload); err != nil {
|
||||
return mongoPositionPayload{}, nil, nil, fmt.Errorf("decode MongoDB CDC position: %w", err)
|
||||
}
|
||||
if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
|
||||
return mongoPositionPayload{}, nil, nil, errors.New("MongoDB CDC position must contain exactly one JSON object")
|
||||
}
|
||||
if payload.Version != mongoPositionVersion {
|
||||
return mongoPositionPayload{}, nil, nil, fmt.Errorf("unsupported MongoDB CDC position version %d", payload.Version)
|
||||
}
|
||||
if strings.TrimSpace(payload.ScopeHash) == "" {
|
||||
return mongoPositionPayload{}, nil, nil, errors.New("MongoDB CDC position scope is required")
|
||||
}
|
||||
hasToken := strings.TrimSpace(payload.ResumeTokenBSON) != ""
|
||||
hasOperationTime := payload.OperationTime != nil
|
||||
if hasToken == hasOperationTime {
|
||||
return mongoPositionPayload{}, nil, nil, errors.New("MongoDB CDC position must contain exactly one resume token or operation time")
|
||||
}
|
||||
if hasToken {
|
||||
if payload.ResumeFormat != mongoResumeTokenFormat {
|
||||
return mongoPositionPayload{}, nil, nil, fmt.Errorf("unsupported MongoDB CDC resume token format %q", payload.ResumeFormat)
|
||||
}
|
||||
raw, err := base64.RawURLEncoding.DecodeString(payload.ResumeTokenBSON)
|
||||
if err != nil {
|
||||
return mongoPositionPayload{}, nil, nil, fmt.Errorf("decode MongoDB CDC resume token: %w", err)
|
||||
}
|
||||
if len(raw) > mongoMaxResumeTokenBytes {
|
||||
return mongoPositionPayload{}, nil, nil, errors.New("MongoDB CDC resume token exceeds the supported size")
|
||||
}
|
||||
token := bson.Raw(raw)
|
||||
if err := token.Validate(); err != nil {
|
||||
return mongoPositionPayload{}, nil, nil, fmt.Errorf("MongoDB CDC resume token is invalid BSON: %w", err)
|
||||
}
|
||||
return payload, token, nil, nil
|
||||
}
|
||||
timestamp := bson.Timestamp{T: payload.OperationTime.Seconds, I: payload.OperationTime.Increment}
|
||||
if timestamp.IsZero() {
|
||||
return mongoPositionPayload{}, nil, nil, errors.New("MongoDB CDC operation time is empty")
|
||||
}
|
||||
return payload, nil, ×tamp, nil
|
||||
}
|
||||
|
||||
func mongoPositionIdentity(position Position) (string, error) {
|
||||
payload, token, operationTime, err := decodeMongoPosition(position)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(token) > 0 {
|
||||
return payload.ScopeHash + ":token:" + base64.RawURLEncoding.EncodeToString(token), nil
|
||||
}
|
||||
return fmt.Sprintf("%s:time:%d:%d", payload.ScopeHash, operationTime.T, operationTime.I), nil
|
||||
}
|
||||
367
internal/synccdc/mongodb_stream.go
Normal file
367
internal/synccdc/mongodb_stream.go
Normal file
@@ -0,0 +1,367 @@
|
||||
package synccdc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.mongodb.org/mongo-driver/v2/bson"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrMongoDBStreamClosed = errors.New("MongoDB CDC stream is closed")
|
||||
ErrMongoDBResnapshotRequired = errors.New("MongoDB CDC stream ended and requires a new snapshot")
|
||||
)
|
||||
|
||||
const mongoMaxBufferedEventsPerDelivery = 256
|
||||
|
||||
type mongoBufferedCursor interface {
|
||||
TryNext(context.Context) bool
|
||||
RemainingBatchLength() int
|
||||
}
|
||||
|
||||
type mongoChangeNamespace struct {
|
||||
Database string `bson:"db"`
|
||||
Collection string `bson:"coll"`
|
||||
}
|
||||
|
||||
type mongoChangeEnvelope struct {
|
||||
OperationType string `bson:"operationType"`
|
||||
Namespace mongoChangeNamespace `bson:"ns"`
|
||||
DocumentKey bson.Raw `bson:"documentKey"`
|
||||
FullDocument bson.Raw `bson:"fullDocument"`
|
||||
ClusterTime bson.Timestamp `bson:"clusterTime"`
|
||||
WallTime time.Time `bson:"wallTime"`
|
||||
SessionID bson.Raw `bson:"lsid"`
|
||||
TxnNumber *int64 `bson:"txnNumber"`
|
||||
}
|
||||
|
||||
type mongoDBStream struct {
|
||||
connection mongoConnection
|
||||
cursor mongoCursor
|
||||
scopeHash string
|
||||
allowed map[string]struct{}
|
||||
|
||||
lifetimeCtx context.Context
|
||||
cancel context.CancelFunc
|
||||
opMu sync.Mutex
|
||||
stateMu sync.Mutex
|
||||
closed bool
|
||||
lastDeliveredIdentity string
|
||||
lastAcknowledgedIdentity string
|
||||
closeOnce sync.Once
|
||||
closeErr error
|
||||
}
|
||||
|
||||
func newMongoDBStream(connection mongoConnection, cursor mongoCursor, namespaces []mongoNamespace, scopeHash string) *mongoDBStream {
|
||||
lifetimeCtx, cancel := context.WithCancel(context.Background())
|
||||
allowed := make(map[string]struct{}, len(namespaces))
|
||||
for _, namespace := range namespaces {
|
||||
allowed[mongoNamespaceKey(namespace.Database, namespace.Collection)] = struct{}{}
|
||||
}
|
||||
return &mongoDBStream{
|
||||
connection: connection,
|
||||
cursor: cursor,
|
||||
scopeHash: scopeHash,
|
||||
allowed: allowed,
|
||||
lifetimeCtx: lifetimeCtx,
|
||||
cancel: cancel,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *mongoDBStream) Next(ctx context.Context) (Transaction, error) {
|
||||
if s == nil {
|
||||
return Transaction{}, ErrMongoDBStreamClosed
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return Transaction{}, err
|
||||
}
|
||||
s.stateMu.Lock()
|
||||
closed := s.closed
|
||||
s.stateMu.Unlock()
|
||||
if closed {
|
||||
return Transaction{}, ErrMongoDBStreamClosed
|
||||
}
|
||||
|
||||
nextCtx, cancel := context.WithCancel(ctx)
|
||||
stop := context.AfterFunc(s.lifetimeCtx, cancel)
|
||||
defer func() {
|
||||
stop()
|
||||
cancel()
|
||||
}()
|
||||
|
||||
s.opMu.Lock()
|
||||
defer s.opMu.Unlock()
|
||||
s.stateMu.Lock()
|
||||
closed = s.closed
|
||||
s.stateMu.Unlock()
|
||||
if closed {
|
||||
return Transaction{}, ErrMongoDBStreamClosed
|
||||
}
|
||||
if !s.cursor.Next(nextCtx) {
|
||||
cursorErr := s.cursor.Err()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return Transaction{}, err
|
||||
}
|
||||
if s.lifetimeCtx.Err() != nil {
|
||||
return Transaction{}, ErrMongoDBStreamClosed
|
||||
}
|
||||
if cursorErr != nil {
|
||||
return Transaction{}, fmt.Errorf("read MongoDB change stream: %w", cursorErr)
|
||||
}
|
||||
return Transaction{}, ErrMongoDBResnapshotRequired
|
||||
}
|
||||
|
||||
event, position, identity, err := s.decodeCurrentEvent()
|
||||
if err != nil {
|
||||
return Transaction{}, err
|
||||
}
|
||||
events := []Event{event}
|
||||
if bufferedCursor, ok := s.cursor.(mongoBufferedCursor); ok {
|
||||
for len(events) < mongoMaxBufferedEventsPerDelivery && bufferedCursor.RemainingBatchLength() > 0 && bufferedCursor.TryNext(nextCtx) {
|
||||
bufferedEvent, bufferedPosition, bufferedIdentity, decodeErr := s.decodeCurrentEvent()
|
||||
if decodeErr != nil {
|
||||
return Transaction{}, decodeErr
|
||||
}
|
||||
events = append(events, bufferedEvent)
|
||||
position = bufferedPosition
|
||||
identity = bufferedIdentity
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return Transaction{}, err
|
||||
}
|
||||
if s.lifetimeCtx.Err() != nil {
|
||||
return Transaction{}, ErrMongoDBStreamClosed
|
||||
}
|
||||
if cursorErr := s.cursor.Err(); cursorErr != nil {
|
||||
return Transaction{}, fmt.Errorf("read MongoDB buffered change stream events: %w", cursorErr)
|
||||
}
|
||||
}
|
||||
s.stateMu.Lock()
|
||||
s.lastDeliveredIdentity = identity
|
||||
s.stateMu.Unlock()
|
||||
return Transaction{Events: events, Position: position}, nil
|
||||
}
|
||||
|
||||
func (s *mongoDBStream) decodeCurrentEvent() (Event, Position, string, error) {
|
||||
var envelope mongoChangeEnvelope
|
||||
if err := s.cursor.Decode(&envelope); err != nil {
|
||||
return Event{}, Position{}, "", fmt.Errorf("decode MongoDB change stream event: %w", err)
|
||||
}
|
||||
event, err := mapMongoChangeEvent(envelope, s.allowed)
|
||||
if err != nil {
|
||||
return Event{}, Position{}, "", err
|
||||
}
|
||||
position, err := mongoResumeTokenPosition(s.scopeHash, append(bson.Raw(nil), s.cursor.ResumeToken()...))
|
||||
if err != nil {
|
||||
return Event{}, Position{}, "", err
|
||||
}
|
||||
identity, err := mongoPositionIdentity(position)
|
||||
if err != nil {
|
||||
return Event{}, Position{}, "", err
|
||||
}
|
||||
return event, position, identity, nil
|
||||
}
|
||||
|
||||
func (s *mongoDBStream) Acknowledge(ctx context.Context, position Position) error {
|
||||
if s == nil {
|
||||
return ErrMongoDBStreamClosed
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
payload, _, _, err := decodeMongoPosition(position)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if payload.ScopeHash != s.scopeHash {
|
||||
return errors.New("MongoDB CDC acknowledgement belongs to a different source or namespace selection")
|
||||
}
|
||||
identity, err := mongoPositionIdentity(position)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.stateMu.Lock()
|
||||
defer s.stateMu.Unlock()
|
||||
if s.closed {
|
||||
return ErrMongoDBStreamClosed
|
||||
}
|
||||
if identity == s.lastAcknowledgedIdentity && identity == s.lastDeliveredIdentity && identity != "" {
|
||||
return nil
|
||||
}
|
||||
if identity == "" || identity != s.lastDeliveredIdentity {
|
||||
return errors.New("MongoDB CDC can only acknowledge the most recently delivered position")
|
||||
}
|
||||
// MongoDB exposes no server acknowledgement for change streams. Recording
|
||||
// this identity only validates the caller's durable local checkpoint.
|
||||
s.lastAcknowledgedIdentity = identity
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *mongoDBStream) Close() error {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
s.closeOnce.Do(func() {
|
||||
s.stateMu.Lock()
|
||||
s.closed = true
|
||||
s.stateMu.Unlock()
|
||||
s.cancel()
|
||||
|
||||
s.opMu.Lock()
|
||||
defer s.opMu.Unlock()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
var cursorErr error
|
||||
if s.cursor != nil {
|
||||
cursorErr = s.cursor.Close(ctx)
|
||||
}
|
||||
var disconnectErr error
|
||||
if s.connection != nil {
|
||||
disconnectErr = s.connection.Disconnect(ctx)
|
||||
}
|
||||
s.closeErr = errors.Join(cursorErr, disconnectErr)
|
||||
})
|
||||
return s.closeErr
|
||||
}
|
||||
|
||||
func mapMongoChangeEvent(envelope mongoChangeEnvelope, allowed map[string]struct{}) (Event, error) {
|
||||
if _, ok := allowed[mongoNamespaceKey(envelope.Namespace.Database, envelope.Namespace.Collection)]; !ok {
|
||||
return Event{}, fmt.Errorf("MongoDB change stream returned out-of-scope namespace %s.%s", envelope.Namespace.Database, envelope.Namespace.Collection)
|
||||
}
|
||||
operation := envelope.OperationType
|
||||
switch operation {
|
||||
case "insert", "replace", "update", "delete":
|
||||
default:
|
||||
return Event{}, fmt.Errorf("unsupported MongoDB change-stream operation %q", operation)
|
||||
}
|
||||
key, err := mongoRawDocumentToJSONMap(envelope.DocumentKey)
|
||||
if err != nil {
|
||||
return Event{}, fmt.Errorf("decode MongoDB change-stream document key: %w", err)
|
||||
}
|
||||
if len(key) == 0 {
|
||||
return Event{}, fmt.Errorf("MongoDB %s event has no document key", operation)
|
||||
}
|
||||
var after map[string]interface{}
|
||||
if operation != "delete" {
|
||||
after, err = mongoRawDocumentToJSONMap(envelope.FullDocument)
|
||||
if err != nil {
|
||||
return Event{}, fmt.Errorf("decode MongoDB %s fullDocument: %w", operation, err)
|
||||
}
|
||||
if len(after) == 0 && operation != "update" {
|
||||
return Event{}, fmt.Errorf("MongoDB %s event has no fullDocument; updateLookup could not produce an apply-safe row", operation)
|
||||
}
|
||||
}
|
||||
commitTime := envelope.WallTime.UTC()
|
||||
if !envelope.ClusterTime.IsZero() {
|
||||
commitTime = time.Unix(int64(envelope.ClusterTime.T), 0).UTC()
|
||||
}
|
||||
if commitTime.IsZero() {
|
||||
return Event{}, errors.New("MongoDB change-stream event has no commit time")
|
||||
}
|
||||
return Event{
|
||||
Object: ObjectRef{
|
||||
Database: envelope.Namespace.Database,
|
||||
Name: envelope.Namespace.Collection,
|
||||
},
|
||||
Operation: operation,
|
||||
Key: key,
|
||||
After: after,
|
||||
CommitTime: commitTime,
|
||||
SourceTxID: mongoSourceTransactionID(envelope.SessionID, envelope.TxnNumber),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func mongoRawDocumentToJSONMap(raw bson.Raw) (map[string]interface{}, error) {
|
||||
if len(raw) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if err := raw.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var document bson.M
|
||||
if err := bson.Unmarshal(raw, &document); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
converted, ok := mongoBSONValueToJSON(document).(map[string]interface{})
|
||||
if !ok {
|
||||
return nil, errors.New("MongoDB BSON document did not decode to an object")
|
||||
}
|
||||
return converted, nil
|
||||
}
|
||||
|
||||
func mongoBSONValueToJSON(value interface{}) interface{} {
|
||||
switch typed := value.(type) {
|
||||
case bson.M:
|
||||
result := make(map[string]interface{}, len(typed))
|
||||
for key, item := range typed {
|
||||
result[key] = mongoBSONValueToJSON(item)
|
||||
}
|
||||
return result
|
||||
case bson.D:
|
||||
result := make(map[string]interface{}, len(typed))
|
||||
for _, item := range typed {
|
||||
result[item.Key] = mongoBSONValueToJSON(item.Value)
|
||||
}
|
||||
return result
|
||||
case bson.A:
|
||||
result := make([]interface{}, len(typed))
|
||||
for index, item := range typed {
|
||||
result[index] = mongoBSONValueToJSON(item)
|
||||
}
|
||||
return result
|
||||
case []interface{}:
|
||||
result := make([]interface{}, len(typed))
|
||||
for index, item := range typed {
|
||||
result[index] = mongoBSONValueToJSON(item)
|
||||
}
|
||||
return result
|
||||
case bson.ObjectID, bson.DateTime, bson.Decimal128, bson.Binary, bson.Regex,
|
||||
bson.Timestamp, bson.MaxKey, bson.MinKey, bson.Undefined, int32, int64, []byte, time.Time:
|
||||
if converted, ok := mongoExtendedJSONValue(typed); ok {
|
||||
return converted
|
||||
}
|
||||
return typed
|
||||
default:
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
func mongoExtendedJSONValue(value interface{}) (interface{}, bool) {
|
||||
payload, err := bson.MarshalExtJSON(bson.M{"v": value}, true, false)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
var wrapped map[string]interface{}
|
||||
if err := json.Unmarshal(payload, &wrapped); err != nil {
|
||||
return nil, false
|
||||
}
|
||||
converted, ok := wrapped["v"]
|
||||
return converted, ok
|
||||
}
|
||||
|
||||
func mongoSourceTransactionID(sessionID bson.Raw, transactionNumber *int64) string {
|
||||
if len(sessionID) == 0 || transactionNumber == nil {
|
||||
return ""
|
||||
}
|
||||
sum := sha256.Sum256(append(append([]byte(nil), sessionID...), []byte(strconv.FormatInt(*transactionNumber, 10))...))
|
||||
return "mongo-tx-" + hex.EncodeToString(sum[:12])
|
||||
}
|
||||
|
||||
func mongoNamespaceKey(database, collection string) string {
|
||||
return database + "\x00" + collection
|
||||
}
|
||||
|
||||
var _ Stream = (*mongoDBStream)(nil)
|
||||
276
internal/synccdc/mongodb_stream_test.go
Normal file
276
internal/synccdc/mongodb_stream_test.go
Normal file
@@ -0,0 +1,276 @@
|
||||
package synccdc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.mongodb.org/mongo-driver/v2/bson"
|
||||
)
|
||||
|
||||
type bufferedFakeMongoCursor struct {
|
||||
*fakeMongoCursor
|
||||
}
|
||||
|
||||
func (cursor *bufferedFakeMongoCursor) TryNext(ctx context.Context) bool {
|
||||
return cursor.fakeMongoCursor.Next(ctx)
|
||||
}
|
||||
|
||||
func (cursor *bufferedFakeMongoCursor) RemainingBatchLength() int {
|
||||
cursor.mu.Lock()
|
||||
defer cursor.mu.Unlock()
|
||||
return len(cursor.events) - cursor.index
|
||||
}
|
||||
|
||||
func TestMongoStreamMapsInsertReplaceUpdateAndDelete(t *testing.T) {
|
||||
objectID := bson.NewObjectID()
|
||||
sessionID := mustMongoRaw(t, bson.D{{Key: "id", Value: bson.Binary{Subtype: 4, Data: []byte("0123456789abcdef")}}})
|
||||
txnNumber := int64(9)
|
||||
operations := []string{"insert", "replace", "update", "delete"}
|
||||
events := make([]bson.Raw, 0, len(operations))
|
||||
tokens := make([]bson.Raw, 0, len(operations))
|
||||
for index, operation := range operations {
|
||||
document := bson.D{
|
||||
{Key: "_id", Value: bson.D{{Key: "_data", Value: operation}}},
|
||||
{Key: "operationType", Value: operation},
|
||||
{Key: "ns", Value: bson.D{{Key: "db", Value: "app"}, {Key: "coll", Value: "orders"}}},
|
||||
{Key: "documentKey", Value: bson.D{{Key: "_id", Value: objectID}}},
|
||||
{Key: "clusterTime", Value: bson.Timestamp{T: uint32(1_725_000_000 + index), I: 1}},
|
||||
{Key: "lsid", Value: sessionID},
|
||||
{Key: "txnNumber", Value: txnNumber},
|
||||
}
|
||||
if operation != "delete" {
|
||||
document = append(document, bson.E{Key: "fullDocument", Value: bson.D{
|
||||
{Key: "_id", Value: objectID},
|
||||
{Key: "amount", Value: int64(9_007_199_254_740_993)},
|
||||
{Key: "ratio", Value: 1.25},
|
||||
}})
|
||||
}
|
||||
events = append(events, mustMongoRaw(t, document))
|
||||
tokens = append(tokens, mustMongoRaw(t, bson.D{{Key: "_data", Value: "token-" + operation}}))
|
||||
}
|
||||
cursor := &fakeMongoCursor{events: events, tokens: tokens}
|
||||
connection := &fakeMongoConnection{cursor: cursor}
|
||||
stream := newMongoDBStream(connection, cursor, []mongoNamespace{{Database: "app", Collection: "orders"}}, "scope-1")
|
||||
defer stream.Close()
|
||||
|
||||
var previousPosition Position
|
||||
for _, expectedOperation := range operations {
|
||||
transaction, err := stream.Next(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("next %s: %v", expectedOperation, err)
|
||||
}
|
||||
if len(transaction.Events) != 1 || transaction.Events[0].Operation != expectedOperation {
|
||||
t.Fatalf("unexpected transaction: %+v", transaction)
|
||||
}
|
||||
event := transaction.Events[0]
|
||||
if event.Object.Database != "app" || event.Object.Name != "orders" {
|
||||
t.Fatalf("unexpected namespace: %+v", event.Object)
|
||||
}
|
||||
if event.Key["_id"] == nil {
|
||||
t.Fatalf("document key was not preserved: %+v", event.Key)
|
||||
}
|
||||
objectIDJSON, ok := event.Key["_id"].(map[string]interface{})
|
||||
if !ok || objectIDJSON["$oid"] != objectID.Hex() {
|
||||
t.Fatalf("ObjectID was not preserved as canonical Extended JSON: %+v", event.Key["_id"])
|
||||
}
|
||||
if expectedOperation == "delete" {
|
||||
if event.After != nil {
|
||||
t.Fatalf("delete after image = %+v", event.After)
|
||||
}
|
||||
} else {
|
||||
amountJSON, ok := event.After["amount"].(map[string]interface{})
|
||||
if !ok || amountJSON["$numberLong"] != "9007199254740993" {
|
||||
t.Fatalf("64-bit integer was not preserved as canonical Extended JSON: %+v", event.After)
|
||||
}
|
||||
if event.After["ratio"] != float64(1.25) {
|
||||
t.Fatalf("ordinary double must match snapshot conversion semantics: %+v", event.After)
|
||||
}
|
||||
}
|
||||
if !strings.HasPrefix(event.SourceTxID, "mongo-tx-") {
|
||||
t.Fatalf("source transaction identity missing: %q", event.SourceTxID)
|
||||
}
|
||||
if previousPosition.Adapter != "" {
|
||||
if err := stream.Acknowledge(context.Background(), previousPosition); err == nil {
|
||||
t.Fatalf("stale acknowledgement before %s must be rejected", expectedOperation)
|
||||
}
|
||||
}
|
||||
if err := stream.Acknowledge(context.Background(), transaction.Position); err != nil {
|
||||
t.Fatalf("acknowledge %s: %v", expectedOperation, err)
|
||||
}
|
||||
if err := stream.Acknowledge(context.Background(), transaction.Position); err != nil {
|
||||
t.Fatalf("idempotent acknowledge %s: %v", expectedOperation, err)
|
||||
}
|
||||
previousPosition = transaction.Position
|
||||
}
|
||||
}
|
||||
|
||||
func TestMongoStreamDrainsBufferedEventsAndAcknowledgesFinalPosition(t *testing.T) {
|
||||
events := make([]bson.Raw, 0, 3)
|
||||
tokens := make([]bson.Raw, 0, 3)
|
||||
for index := 1; index <= 3; index++ {
|
||||
events = append(events, mustMongoRaw(t, bson.D{
|
||||
{Key: "operationType", Value: "insert"},
|
||||
{Key: "ns", Value: bson.D{{Key: "db", Value: "app"}, {Key: "coll", Value: "orders"}}},
|
||||
{Key: "documentKey", Value: bson.D{{Key: "_id", Value: index}}},
|
||||
{Key: "fullDocument", Value: bson.D{{Key: "_id", Value: index}, {Key: "value", Value: index}}},
|
||||
{Key: "clusterTime", Value: bson.Timestamp{T: uint32(100 + index), I: 1}},
|
||||
}))
|
||||
tokens = append(tokens, mustMongoRaw(t, bson.D{{Key: "_data", Value: fmt.Sprintf("token-%d", index)}}))
|
||||
}
|
||||
cursor := &bufferedFakeMongoCursor{fakeMongoCursor: &fakeMongoCursor{events: events, tokens: tokens}}
|
||||
stream := newMongoDBStream(&fakeMongoConnection{cursor: cursor}, cursor, []mongoNamespace{{Database: "app", Collection: "orders"}}, "scope-1")
|
||||
defer stream.Close()
|
||||
|
||||
transaction, err := stream.Next(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("next buffered transaction: %v", err)
|
||||
}
|
||||
if len(transaction.Events) != 3 {
|
||||
t.Fatalf("buffered events = %d, want 3", len(transaction.Events))
|
||||
}
|
||||
for index, event := range transaction.Events {
|
||||
encodedID, ok := event.Key["_id"].(map[string]interface{})
|
||||
if !ok || encodedID["$numberInt"] != fmt.Sprint(index+1) {
|
||||
t.Fatalf("event %d key = %#v", index, event.Key)
|
||||
}
|
||||
}
|
||||
intermediate, err := mongoResumeTokenPosition("scope-1", tokens[0])
|
||||
if err != nil {
|
||||
t.Fatalf("build intermediate position: %v", err)
|
||||
}
|
||||
if err := stream.Acknowledge(context.Background(), intermediate); err == nil {
|
||||
t.Fatal("buffered delivery accepted a non-final position")
|
||||
}
|
||||
if err := stream.Acknowledge(context.Background(), transaction.Position); err != nil {
|
||||
t.Fatalf("acknowledge final buffered position: %v", err)
|
||||
}
|
||||
expectedPosition, err := mongoResumeTokenPosition("scope-1", tokens[2])
|
||||
if err != nil {
|
||||
t.Fatalf("build final position: %v", err)
|
||||
}
|
||||
expectedIdentity, _ := mongoPositionIdentity(expectedPosition)
|
||||
actualIdentity, _ := mongoPositionIdentity(transaction.Position)
|
||||
if actualIdentity != expectedIdentity {
|
||||
t.Fatalf("transaction position = %s, want final %s", actualIdentity, expectedIdentity)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMongoStreamRejectsOutOfScopeEvent(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
raw bson.Raw
|
||||
error string
|
||||
}{
|
||||
{
|
||||
name: "out of scope",
|
||||
raw: mustMongoRaw(t, bson.D{
|
||||
{Key: "operationType", Value: "delete"},
|
||||
{Key: "ns", Value: bson.D{{Key: "db", Value: "other"}, {Key: "coll", Value: "orders"}}},
|
||||
{Key: "documentKey", Value: bson.D{{Key: "_id", Value: 1}}},
|
||||
{Key: "clusterTime", Value: bson.Timestamp{T: 100, I: 1}},
|
||||
}),
|
||||
error: "out-of-scope",
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cursor := &fakeMongoCursor{
|
||||
events: []bson.Raw{test.raw},
|
||||
tokens: []bson.Raw{mustMongoRaw(t, bson.D{{Key: "_data", Value: "token"}})},
|
||||
}
|
||||
stream := newMongoDBStream(&fakeMongoConnection{}, cursor, []mongoNamespace{{Database: "app", Collection: "orders"}}, "scope-1")
|
||||
defer stream.Close()
|
||||
if _, err := stream.Next(context.Background()); err == nil || !strings.Contains(err.Error(), test.error) {
|
||||
t.Fatalf("error = %v, want substring %q", err, test.error)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMongoStreamDeliversUpdateLookupNullWithCheckpoint(t *testing.T) {
|
||||
cursor := &fakeMongoCursor{
|
||||
events: []bson.Raw{mustMongoRaw(t, bson.D{
|
||||
{Key: "operationType", Value: "update"},
|
||||
{Key: "ns", Value: bson.D{{Key: "db", Value: "app"}, {Key: "coll", Value: "orders"}}},
|
||||
{Key: "documentKey", Value: bson.D{{Key: "_id", Value: 1}}},
|
||||
{Key: "clusterTime", Value: bson.Timestamp{T: 100, I: 1}},
|
||||
})},
|
||||
tokens: []bson.Raw{mustMongoRaw(t, bson.D{{Key: "_data", Value: "update-without-full-document"}})},
|
||||
}
|
||||
stream := newMongoDBStream(&fakeMongoConnection{}, cursor, []mongoNamespace{{Database: "app", Collection: "orders"}}, "scope-1")
|
||||
defer stream.Close()
|
||||
transaction, err := stream.Next(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("deliver updateLookup-null event: %v", err)
|
||||
}
|
||||
if len(transaction.Events) != 1 || transaction.Events[0].Operation != "update" || transaction.Events[0].After != nil {
|
||||
t.Fatalf("unexpected tombstone event: %+v", transaction)
|
||||
}
|
||||
if transaction.Position.Adapter == "" {
|
||||
t.Fatal("updateLookup-null event must carry a resumable checkpoint")
|
||||
}
|
||||
if err := stream.Acknowledge(context.Background(), transaction.Position); err != nil {
|
||||
t.Fatalf("acknowledge updateLookup-null checkpoint: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMongoStreamNextIsCancelableAndCloseIsIdempotent(t *testing.T) {
|
||||
cursor := &fakeMongoCursor{block: true, nextStarted: make(chan struct{}, 1)}
|
||||
connection := &fakeMongoConnection{}
|
||||
stream := newMongoDBStream(connection, cursor, []mongoNamespace{{Database: "app", Collection: "orders"}}, "scope-1")
|
||||
result := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := stream.Next(context.Background())
|
||||
result <- err
|
||||
}()
|
||||
select {
|
||||
case <-cursor.nextStarted:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Next did not reach the change-stream cursor")
|
||||
}
|
||||
if err := stream.Close(); err != nil {
|
||||
t.Fatalf("close stream: %v", err)
|
||||
}
|
||||
select {
|
||||
case err := <-result:
|
||||
if !errors.Is(err, ErrMongoDBStreamClosed) {
|
||||
t.Fatalf("Next error after Close = %v", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Close did not cancel blocked Next")
|
||||
}
|
||||
if err := stream.Close(); err != nil {
|
||||
t.Fatalf("second close: %v", err)
|
||||
}
|
||||
cursor.mu.Lock()
|
||||
closeCount := cursor.closed
|
||||
cursor.mu.Unlock()
|
||||
connection.mu.Lock()
|
||||
disconnectCount := connection.disconnects
|
||||
connection.mu.Unlock()
|
||||
if closeCount != 1 || disconnectCount != 1 {
|
||||
t.Fatalf("close count=%d disconnect count=%d", closeCount, disconnectCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMongoStreamCallerCancellationAndAcknowledgementValidation(t *testing.T) {
|
||||
cursor := &fakeMongoCursor{block: true}
|
||||
stream := newMongoDBStream(&fakeMongoConnection{}, cursor, []mongoNamespace{{Database: "app", Collection: "orders"}}, "scope-1")
|
||||
defer stream.Close()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := stream.Next(ctx); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("cancelled Next error = %v", err)
|
||||
}
|
||||
foreign, err := mongoOperationTimePosition("other-scope", bson.Timestamp{T: 1, I: 1})
|
||||
if err != nil {
|
||||
t.Fatalf("build foreign position: %v", err)
|
||||
}
|
||||
if err := stream.Acknowledge(context.Background(), foreign); err == nil || !strings.Contains(err.Error(), "different source") {
|
||||
t.Fatalf("foreign acknowledgement error = %v", err)
|
||||
}
|
||||
}
|
||||
119
internal/synccdc/registry.go
Normal file
119
internal/synccdc/registry.go
Normal file
@@ -0,0 +1,119 @@
|
||||
package synccdc
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type Registry struct {
|
||||
mu sync.RWMutex
|
||||
byName map[string]Adapter
|
||||
bySource map[string]string
|
||||
}
|
||||
|
||||
func NewRegistry() *Registry {
|
||||
registry := &Registry{
|
||||
byName: make(map[string]Adapter),
|
||||
bySource: make(map[string]string),
|
||||
}
|
||||
if err := registry.Register(NewMongoDBAdapter()); err != nil {
|
||||
panic(fmt.Sprintf("register built-in MongoDB CDC adapter: %v", err))
|
||||
}
|
||||
return registry
|
||||
}
|
||||
|
||||
func (r *Registry) Register(adapter Adapter) error {
|
||||
if r == nil {
|
||||
return fmt.Errorf("CDC registry is nil")
|
||||
}
|
||||
if adapter == nil {
|
||||
return fmt.Errorf("CDC adapter is nil")
|
||||
}
|
||||
name := normalize(adapter.Name())
|
||||
if name == "" {
|
||||
return fmt.Errorf("CDC adapter name is required")
|
||||
}
|
||||
sources := adapter.SourceTypes()
|
||||
if len(sources) == 0 {
|
||||
return fmt.Errorf("CDC adapter %s has no source types", name)
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if _, exists := r.byName[name]; exists {
|
||||
return fmt.Errorf("CDC adapter %s is already registered", name)
|
||||
}
|
||||
for _, source := range sources {
|
||||
normalizedSource := normalizeSourceType(source)
|
||||
if normalizedSource == "" {
|
||||
return fmt.Errorf("CDC adapter %s has an empty source type", name)
|
||||
}
|
||||
if existing, exists := r.bySource[normalizedSource]; exists {
|
||||
return fmt.Errorf("CDC source type %s is already handled by %s", normalizedSource, existing)
|
||||
}
|
||||
}
|
||||
r.byName[name] = adapter
|
||||
for _, source := range sources {
|
||||
r.bySource[normalizeSourceType(source)] = name
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Registry) Get(name string) (Adapter, error) {
|
||||
if r == nil {
|
||||
return nil, ErrAdapterNotRegistered
|
||||
}
|
||||
r.mu.RLock()
|
||||
adapter := r.byName[normalize(name)]
|
||||
r.mu.RUnlock()
|
||||
if adapter == nil {
|
||||
return nil, fmt.Errorf("%w: %s", ErrAdapterNotRegistered, strings.TrimSpace(name))
|
||||
}
|
||||
return adapter, nil
|
||||
}
|
||||
|
||||
func (r *Registry) ResolveSource(sourceType string) (Adapter, error) {
|
||||
if r == nil {
|
||||
return nil, ErrAdapterNotRegistered
|
||||
}
|
||||
r.mu.RLock()
|
||||
name := r.bySource[normalizeSourceType(sourceType)]
|
||||
adapter := r.byName[name]
|
||||
r.mu.RUnlock()
|
||||
if adapter == nil {
|
||||
return nil, fmt.Errorf("%w: source type %s", ErrAdapterNotRegistered, strings.TrimSpace(sourceType))
|
||||
}
|
||||
return adapter, nil
|
||||
}
|
||||
|
||||
func (r *Registry) Names() []string {
|
||||
if r == nil {
|
||||
return []string{}
|
||||
}
|
||||
r.mu.RLock()
|
||||
names := make([]string, 0, len(r.byName))
|
||||
for name := range r.byName {
|
||||
names = append(names, name)
|
||||
}
|
||||
r.mu.RUnlock()
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
func normalize(value string) string {
|
||||
return strings.ToLower(strings.TrimSpace(value))
|
||||
}
|
||||
|
||||
func normalizeSourceType(value string) string {
|
||||
switch normalize(value) {
|
||||
case "postgresql", "kingbase", "highgo", "vastbase", "opengauss", "gaussdb":
|
||||
return "postgres"
|
||||
case "mariadb", "oceanbase", "goldendb":
|
||||
return "mysql"
|
||||
case "mongo", "mongodb-v1", "mongodbv1":
|
||||
return "mongodb"
|
||||
default:
|
||||
return normalize(value)
|
||||
}
|
||||
}
|
||||
72
internal/synccdc/registry_test.go
Normal file
72
internal/synccdc/registry_test.go
Normal file
@@ -0,0 +1,72 @@
|
||||
package synccdc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"GoNavi-Wails/internal/connection"
|
||||
)
|
||||
|
||||
type testAdapter struct {
|
||||
name string
|
||||
sources []string
|
||||
}
|
||||
|
||||
func (a testAdapter) Name() string { return a.name }
|
||||
func (a testAdapter) SourceTypes() []string { return a.sources }
|
||||
func (a testAdapter) Probe(context.Context, connection.ConnectionConfig) (Capability, error) {
|
||||
return Capability{Adapter: a.name, Supported: true, Ready: true}, nil
|
||||
}
|
||||
func (a testAdapter) BeginSnapshot(context.Context, Request) (Barrier, error) {
|
||||
return Barrier{}, nil
|
||||
}
|
||||
func (a testAdapter) Open(context.Context, Request, Position) (Stream, error) { return nil, nil }
|
||||
|
||||
func TestRegistryNormalizesSourceFamiliesAndRejectsDuplicates(t *testing.T) {
|
||||
registry := NewRegistry()
|
||||
if err := registry.Register(testAdapter{name: "postgres-logical", sources: []string{"postgres"}}); err != nil {
|
||||
t.Fatalf("register adapter: %v", err)
|
||||
}
|
||||
adapter, err := registry.ResolveSource("openGauss")
|
||||
if err != nil {
|
||||
t.Fatalf("resolve postgres family: %v", err)
|
||||
}
|
||||
if adapter.Name() != "postgres-logical" {
|
||||
t.Fatalf("resolved adapter = %q", adapter.Name())
|
||||
}
|
||||
if err := registry.Register(testAdapter{name: "other", sources: []string{"postgresql"}}); err == nil {
|
||||
t.Fatal("duplicate normalized source type must be rejected")
|
||||
}
|
||||
if _, err := registry.ResolveSource("oracle"); !errors.Is(err, ErrAdapterNotRegistered) {
|
||||
t.Fatalf("missing adapter error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryIncludesBuiltInMongoDBChangeStreamAdapter(t *testing.T) {
|
||||
registry := NewRegistry()
|
||||
for _, sourceType := range []string{"mongodb", "mongodb-v1", "mongo"} {
|
||||
adapter, err := registry.ResolveSource(sourceType)
|
||||
if err != nil {
|
||||
t.Fatalf("resolve %s: %v", sourceType, err)
|
||||
}
|
||||
if adapter.Name() != mongoDBAdapterName {
|
||||
t.Fatalf("resolve %s = %q", sourceType, adapter.Name())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatePositionBindsOpaqueOffsetToAdapter(t *testing.T) {
|
||||
valid := Position{Adapter: "mysql-binlog", Opaque: json.RawMessage(`{"file":"mysql.000001","position":4}`)}
|
||||
if err := ValidatePosition(valid, "mysql-binlog"); err != nil {
|
||||
t.Fatalf("valid position rejected: %v", err)
|
||||
}
|
||||
if err := ValidatePosition(valid, "postgres-logical"); err == nil {
|
||||
t.Fatal("cross-adapter position must be rejected")
|
||||
}
|
||||
valid.Opaque = json.RawMessage(`not-json`)
|
||||
if err := ValidatePosition(valid, "mysql-binlog"); err == nil {
|
||||
t.Fatal("invalid opaque position must be rejected")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user