feat(kafka): 新增 Kafka 数据源连接支持

Refs #387
This commit is contained in:
Syngnat
2026-06-13 21:11:08 +08:00
parent d2f68acae8
commit 0ff17dc27c
37 changed files with 1989 additions and 12 deletions

View File

@@ -18,6 +18,8 @@ describe('connectionDriverType', () => {
expect(normalizeDriverType('qdrant-db')).toBe('qdrant');
expect(normalizeDriverType('apache-iotdb')).toBe('iotdb');
expect(normalizeDriverType('apache_iotdb')).toBe('iotdb');
expect(normalizeDriverType('apache-kafka')).toBe('kafka');
expect(normalizeDriverType('apache_kafka')).toBe('kafka');
expect(normalizeDriverType('doris')).toBe('diros');
expect(normalizeDriverType('open-gauss')).toBe('opengauss');
expect(normalizeDriverType('gauss-db')).toBe('gaussdb');

View File

@@ -18,6 +18,7 @@ export const normalizeDriverType = (value: string): string => {
if (normalized === 'chromadb' || normalized === 'chroma-db') return 'chroma';
if (normalized === 'qdrantdb' || normalized === 'qdrant-db') return 'qdrant';
if (normalized === 'apache-iotdb' || normalized === 'apache_iotdb') return 'iotdb';
if (normalized === 'apache-kafka' || normalized === 'apache_kafka') return 'kafka';
if (normalized === 'doris') return 'diros';
if (
normalized === 'open_gauss' ||

View File

@@ -93,6 +93,7 @@ describe('connectionModalPresentation', () => {
'redis',
'tdengine',
'iotdb',
'kafka',
'custom',
'jvm',
];
@@ -192,6 +193,15 @@ describe('connectionModalPresentation', () => {
'credentials',
'databaseScope',
]);
expect(resolveConnectionConfigLayout('kafka').sections).toEqual([
'identity',
'uri',
'target',
'connectionMode',
'replica',
'service',
'credentials',
]);
});
it('uses localized labels for layout kinds shown in the modal', () => {

View File

@@ -282,6 +282,20 @@ export const resolveConnectionConfigLayout = (
],
};
}
if (type === 'kafka') {
return {
kind: 'generic-sql',
sections: [
'identity',
'uri',
'target',
'connectionMode',
'replica',
'service',
'credentials',
],
};
}
if (postgresCompatibleTypes.has(type)) {
return {
kind: 'postgres-compatible',

View File

@@ -31,6 +31,7 @@ describe('connectionTypeCapabilities', () => {
expect(supportsSSLForType('gaussdb')).toBe(true);
expect(supportsSSLForType('chroma')).toBe(true);
expect(supportsSSLForType('qdrant')).toBe(true);
expect(supportsSSLForType('kafka')).toBe(true);
expect(supportsSSLForType('tdengine')).toBe(true);
expect(supportsSSLForType('iotdb')).toBe(false);
expect(supportsSSLForType('dameng')).toBe(true);
@@ -50,6 +51,8 @@ describe('connectionTypeCapabilities', () => {
expect(supportsSSLClientCertificateForType('chroma')).toBe(false);
expect(supportsSSLCAPathForType('qdrant')).toBe(true);
expect(supportsSSLClientCertificateForType('qdrant')).toBe(false);
expect(supportsSSLCAPathForType('kafka')).toBe(true);
expect(supportsSSLClientCertificateForType('kafka')).toBe(true);
});
it('detects postgres-compatible SSL parameter dialects', () => {
@@ -82,6 +85,7 @@ describe('connectionTypeCapabilities', () => {
expect(supportsConnectionParamsForType('elasticsearch')).toBe(true);
expect(supportsConnectionParamsForType('chroma')).toBe(true);
expect(supportsConnectionParamsForType('qdrant')).toBe(true);
expect(supportsConnectionParamsForType('kafka')).toBe(true);
expect(supportsConnectionParamsForType('redis')).toBe(false);
expect(supportsConnectionParamsForType('sqlite')).toBe(false);
expect(supportsConnectionParamsForType('jvm')).toBe(false);

View File

@@ -47,6 +47,7 @@ const sslSupportedTypes = new Set([
"elasticsearch",
"chroma",
"qdrant",
"kafka",
]);
export const supportsSSLForType = (type: string) =>
@@ -72,6 +73,7 @@ const sslCAPathSupportedTypes = new Set([
"elasticsearch",
"chroma",
"qdrant",
"kafka",
]);
const sslClientCertificateSupportedTypes = new Set([
@@ -91,6 +93,7 @@ const sslClientCertificateSupportedTypes = new Set([
"gaussdb",
"mongodb",
"redis",
"kafka",
]);
export const supportsSSLCAPathForType = (type: string) =>
@@ -139,4 +142,5 @@ export const supportsConnectionParamsForType = (type: string) =>
type === "iotdb" ||
type === "elasticsearch" ||
type === "chroma" ||
type === "qdrant";
type === "qdrant" ||
type === "kafka";

View File

@@ -15,6 +15,7 @@ describe('connectionTypeCatalog', () => {
'NoSQL',
'向量数据库',
'时序数据库',
'消息队列',
'其他',
]);
@@ -28,6 +29,7 @@ describe('connectionTypeCatalog', () => {
expect(keys).toContain('chroma');
expect(keys).toContain('qdrant');
expect(keys).toContain('iotdb');
expect(keys).toContain('kafka');
expect(keys).toContain('jvm');
expect(keys).toContain('custom');
expect(new Set(keys).size).toBe(keys.length);
@@ -46,6 +48,7 @@ describe('connectionTypeCatalog', () => {
expect(getConnectionTypeDefaultPort('chroma')).toBe(8000);
expect(getConnectionTypeDefaultPort('qdrant')).toBe(6333);
expect(getConnectionTypeDefaultPort('iotdb')).toBe(6667);
expect(getConnectionTypeDefaultPort('kafka')).toBe(9092);
expect(getConnectionTypeDefaultPort('sqlite')).toBe(0);
expect(getConnectionTypeDefaultPort('duckdb')).toBe(0);
expect(getConnectionTypeDefaultPort('unknown')).toBe(3306);
@@ -58,6 +61,7 @@ describe('connectionTypeCatalog', () => {
expect(getConnectionTypeHint('chroma')).toContain('向量');
expect(getConnectionTypeHint('qdrant')).toContain('Payload');
expect(getConnectionTypeHint('iotdb')).toContain('Timeseries');
expect(getConnectionTypeHint('kafka')).toContain('Consumer Group');
expect(getConnectionTypeHint('oceanbase')).toBe('MySQL / Oracle 租户');
expect(getConnectionTypeHint('duckdb')).toBe('本地文件连接');
expect(getConnectionTypeHint('mysql')).toBe('标准连接配置');

View File

@@ -60,6 +60,12 @@ export const CONNECTION_TYPE_GROUPS: ConnectionTypeCatalogGroup[] = [
{ key: 'iotdb', name: 'Apache IoTDB' },
],
},
{
label: '消息队列',
items: [
{ key: 'kafka', name: 'Kafka' },
],
},
{
label: '其他',
items: [
@@ -113,6 +119,8 @@ export const getConnectionTypeDefaultPort = (type: string): number => {
return 8000;
case 'qdrant':
return 6333;
case 'kafka':
return 9092;
case 'highgo':
return 5866;
case 'mariadb':
@@ -145,6 +153,8 @@ export const getConnectionTypeHint = (type: string): string => {
return 'Collection 浏览、向量搜索和 Payload 过滤';
case 'iotdb':
return 'Storage Group / Device / Timeseries';
case 'kafka':
return 'Broker / Topic / Consumer Group';
case 'oceanbase':
return 'MySQL / Oracle 租户';
case 'sqlite':

View File

@@ -146,6 +146,24 @@ describe('dataSourceCapabilities', () => {
});
});
it('treats Kafka as a queryable read-only messaging datasource', () => {
expect(getDataSourceCapabilities({ type: 'kafka' })).toMatchObject({
type: 'kafka',
supportsQueryEditor: true,
supportsSqlQueryExport: false,
supportsCopyInsert: false,
supportsCreateDatabase: false,
supportsRenameDatabase: false,
supportsDropDatabase: false,
forceReadOnlyQueryResult: true,
});
expect(getDataSourceCapabilities({ type: 'custom', driver: 'apache-kafka' })).toMatchObject({
type: 'kafka',
supportsQueryEditor: true,
forceReadOnlyQueryResult: true,
});
});
it('treats OceanBase Oracle protocol as Oracle capabilities', () => {
expect(getDataSourceCapabilities({
type: 'oceanbase',

View File

@@ -34,6 +34,10 @@ const normalizeDataSourceToken = (raw: string): string => {
case 'apache-iotdb':
case 'apache_iotdb':
return 'iotdb';
case 'kafka':
case 'apache-kafka':
case 'apache_kafka':
return 'kafka';
case 'intersystems':
case 'intersystemsiris':
case 'inter-systems':
@@ -107,7 +111,7 @@ const COPY_INSERT_TYPES = new Set([
]);
const QUERY_EDITOR_DISABLED_TYPES = new Set(['redis']);
const FORCE_READ_ONLY_QUERY_TYPES = new Set(['tdengine', 'iotdb', 'clickhouse']);
const FORCE_READ_ONLY_QUERY_TYPES = new Set(['tdengine', 'iotdb', 'clickhouse', 'kafka']);
const MANUAL_TOTAL_COUNT_TYPES = new Set(['duckdb', 'oracle']);
const APPROXIMATE_TABLE_COUNT_TYPES = new Set(['duckdb', 'oracle']);
const APPROXIMATE_TOTAL_PAGE_TYPES = new Set(['duckdb']);

View File

@@ -6,4 +6,8 @@ describe('buildTableSelectQuery', () => {
it('quotes uppercase postgres table names in new query templates', () => {
expect(buildTableSelectQuery('postgres', 'public.MyTable')).toBe('SELECT * FROM public."MyTable";');
});
it('adds a preview limit for Kafka topic browsing', () => {
expect(buildTableSelectQuery('kafka', 'logs.app-1')).toBe('SELECT * FROM "logs.app-1" LIMIT 100;');
});
});

View File

@@ -5,5 +5,8 @@ export const buildTableSelectQuery = (dbType: string, tableName: string): string
if (!normalizedTableName) {
return 'SELECT * FROM ';
}
if (String(dbType || '').trim().toLowerCase() === 'kafka') {
return `SELECT * FROM ${quoteQualifiedIdent(dbType, normalizedTableName)} LIMIT 100;`;
}
return `SELECT * FROM ${quoteQualifiedIdent(dbType, normalizedTableName)};`;
};

View File

@@ -59,6 +59,11 @@ describe('quoteQualifiedIdent', () => {
.toBe('`root`.`sg`.`d1`');
});
it('keeps Kafka topic names as one quoted identifier', () => {
expect(quoteQualifiedIdent('kafka', 'logs.app-1'))
.toBe('"logs.app-1"');
});
it('does not split dots inside quoted DuckDB identifiers', () => {
expect(quoteQualifiedIdent('duckdb', '"daily.events"."2026.06"'))
.toBe('"daily.events"."2026.06"');

View File

@@ -54,6 +54,9 @@ export const quoteIdentPart = (dbType: string, ident: string) => {
export const quoteQualifiedIdent = (dbType: string, ident: string) => {
const raw = (ident || '').trim();
if (!raw) return raw;
if ((dbType || '').trim().toLowerCase() === 'kafka') {
return quoteIdentPart(dbType, raw);
}
const parts = splitQualifiedNameSegments(raw).filter(Boolean);
if (parts.length === 0) return quoteIdentPart(dbType, raw);
if (parts.length === 1 && parts[0] === normalizeIdentPart(raw)) return quoteIdentPart(dbType, raw);

View File

@@ -38,6 +38,8 @@ describe('sqlDialect', () => {
expect(resolveSqlDialect('custom', 'qdrant-db')).toBe('qdrant');
expect(resolveSqlDialect('Apache-IoTDB')).toBe('iotdb');
expect(resolveSqlDialect('custom', 'apache_iotdb')).toBe('iotdb');
expect(resolveSqlDialect('Apache-Kafka')).toBe('kafka');
expect(resolveSqlDialect('custom', 'apache_kafka')).toBe('kafka');
expect(resolveSqlDialect('OceanBase', '', { oceanBaseProtocol: 'oracle' })).toBe('oracle');
expect(resolveSqlDialect('custom', 'oceanbase', { oceanBaseProtocol: 'oracle' })).toBe('oracle');
expect(isMysqlFamilyDialect('mariadb')).toBe(true);
@@ -71,6 +73,11 @@ describe('sqlDialect', () => {
expect(resolveSqlKeywords('iotdb')).not.toEqual(expect.arrayContaining(['TAGS', 'USING']));
});
it('resolves Kafka completion keywords for topic discovery and consume syntax', () => {
expect(resolveSqlKeywords('kafka')).toEqual(expect.arrayContaining(['SHOW TOPICS', 'DESCRIBE TOPIC', 'CONSUME']));
expect(resolveSqlKeywords('kafka')).not.toEqual(expect.arrayContaining(['ALIGN BY DEVICE', 'AUTO_INCREMENT']));
});
it('resolves GaussDB completion keywords and functions as a PostgreSQL-like dialect', () => {
expect(resolveSqlKeywords('gaussdb')).toEqual(expect.arrayContaining(['RETURNING', 'SERIAL', 'JSONB']));
expect(names(resolveSqlFunctions('gaussdb'))).toEqual(expect.arrayContaining(['STRING_AGG', 'TO_CHAR', 'CURRENT_DATABASE']));

View File

@@ -29,6 +29,7 @@ export type SqlDialect =
| 'clickhouse'
| 'tdengine'
| 'iotdb'
| 'kafka'
| 'mongodb'
| 'redis'
| 'elasticsearch'
@@ -135,6 +136,10 @@ export const resolveSqlDialect = (
case 'apache-iotdb':
case 'apache_iotdb':
return 'iotdb';
case 'kafka':
case 'apache-kafka':
case 'apache_kafka':
return 'kafka';
default:
break;
}
@@ -159,6 +164,7 @@ export const resolveSqlDialect = (
if (source.includes('clickhouse')) return 'clickhouse';
if (source.includes('tdengine')) return 'tdengine';
if (source.includes('iotdb')) return 'iotdb';
if (source.includes('kafka')) return 'kafka';
if (source.includes('sqlserver') || source.includes('mssql')) return 'sqlserver';
if (source.includes('iris') || source.includes('intersystems')) return 'iris';
if (source.includes('elastic')) return 'elasticsearch';
@@ -611,6 +617,17 @@ const IOTDB_KEYWORDS = [
'COMPRESSION',
];
const KAFKA_KEYWORDS = [
'SHOW TOPICS',
'SHOW TOPIC',
'DESCRIBE TOPIC',
'CONSUME',
'GROUP',
'FROM',
'LIMIT',
'OFFSET',
];
export const resolveSqlKeywords = (dbType: string): string[] => {
const dialect = resolveSqlDialect(dbType);
if (dialect === 'starrocks') return unique([...COMMON_KEYWORDS, ...MYSQL_KEYWORDS, ...STARROCKS_KEYWORDS]);
@@ -623,6 +640,7 @@ export const resolveSqlKeywords = (dbType: string): string[] => {
if (dialect === 'clickhouse') return unique([...COMMON_KEYWORDS, ...CLICKHOUSE_KEYWORDS]);
if (dialect === 'tdengine') return unique([...COMMON_KEYWORDS, ...TDENGINE_KEYWORDS]);
if (dialect === 'iotdb') return unique([...COMMON_KEYWORDS, ...IOTDB_KEYWORDS]);
if (dialect === 'kafka') return unique([...COMMON_KEYWORDS, ...KAFKA_KEYWORDS]);
return COMMON_KEYWORDS;
};