refactor: store mail flags in sparse relation table

2026-09-07 00:17:12 +08:00 · 2026-08-27 14:03:50 +08:00
parent 523f9b8af6
commit f12163490e
25 changed files with 750 additions and 179 deletions
@@ -1,13 +1,11 @@
-import { getBooleanValue } from './utils';
-
-export const MAIL_FLAGS = {
-    UNREAD: 1 << 0,
-    ANSWERED: 1 << 1,
-    FLAGGED: 1 << 2,
-    DELETED: 1 << 3,
-    DRAFT: 1 << 4,
-    JUNK: 1 << 5,
-} as const;
+export enum MailFlag {
+    UNREAD = 0,
+    ANSWERED = 1,
+    FLAGGED = 2,
+    DELETED = 3,
+    DRAFT = 4,
+    JUNK = 5,
+}

 export const CUSTOM_MAIL_FLAG_OFFSET = 10;
 export const CUSTOM_MAIL_FLAG_COUNT = 10;
@@ -26,9 +24,14 @@ export type MailStateOption = {
    default?: boolean;
 };

+type MailStateMutation = {
+    add?: number;
+    remove?: number[];
+};
+
 export type MailStateDefinition = MailStateOption & {
-    filter?: { mask: number; set: boolean };
-    mutation?: { add: number; remove: number };
+    filter?: { flag: number; present: boolean };
+    mutation?: MailStateMutation;
 };

 const SYSTEM_MAIL_STATES: MailStateDefinition[] = [
@@ -37,15 +40,15 @@ const SYSTEM_MAIL_STATES: MailStateDefinition[] = [
        value: MailState.UNREAD,
        label_key: 'unread',
        unread: true,
-        filter: { mask: MAIL_FLAGS.UNREAD, set: true },
-        mutation: { add: MAIL_FLAGS.UNREAD, remove: 0 },
+        filter: { flag: MailFlag.UNREAD, present: true },
+        mutation: { add: MailFlag.UNREAD },
    },
    {
        value: MailState.READ,
        label_key: 'read',
        unread: false,
-        filter: { mask: MAIL_FLAGS.UNREAD, set: false },
-        mutation: { add: 0, remove: MAIL_FLAGS.UNREAD },
+        filter: { flag: MailFlag.UNREAD, present: false },
+        mutation: { remove: [MailFlag.UNREAD] },
    },
 ];

@@ -68,9 +71,9 @@ const getMailStateDefinition = (

 export const getCustomMailFlag = (slot: number): number => {
    if (!Number.isInteger(slot) || slot < 0 || slot >= CUSTOM_MAIL_FLAG_COUNT) {
-        throw new Error("Invalid custom mail flag slot");
+        throw new Error('Invalid custom mail flag slot');
    }
-    return 1 << (CUSTOM_MAIL_FLAG_OFFSET + slot);
+    return CUSTOM_MAIL_FLAG_OFFSET + slot;
 };

 export type CustomMailStateConfig = {
@@ -78,7 +81,10 @@ export type CustomMailStateConfig = {
    name: string;
 };

-const CUSTOM_MAIL_FLAGS_MASK = ((1 << CUSTOM_MAIL_FLAG_COUNT) - 1) << CUSTOM_MAIL_FLAG_OFFSET;
+const CUSTOM_MAIL_FLAGS = Array.from(
+    { length: CUSTOM_MAIL_FLAG_COUNT },
+    (_, slot) => getCustomMailFlag(slot),
+);

 export const createCustomMailStateDefinitions = (
    configs: CustomMailStateConfig[],
@@ -88,23 +94,58 @@ export const createCustomMailStateDefinitions = (
        return {
            value: `custom:${config.slot}`,
            label: config.name,
-            filter: { mask: flag, set: true },
-            mutation: { add: flag, remove: CUSTOM_MAIL_FLAGS_MASK & ~flag },
+            filter: { flag, present: true },
+            mutation: { add: flag, remove: CUSTOM_MAIL_FLAGS.filter(value => value !== flag) },
        };
    });
 };

-export const serializeMailState = <T extends Record<string, unknown>>(
+const withoutLegacyFlags = <T extends Record<string, unknown>>(row: T): T => {
+    const result = { ...row };
+    delete result.flags;
+    return result;
+};
+
+const isMailFlagsEnabled = (env: Bindings): boolean => {
+    return env.ENABLE_MAIL_FLAGS === true || env.ENABLE_MAIL_FLAGS === 'true';
+};
+
+export const serializeMailStates = async <T extends Record<string, unknown>>(
+    db: D1Database,
+    rows: T[],
+    env: Bindings,
+): Promise<T[]> => {
+    const results = rows.map(withoutLegacyFlags);
+    if (!isMailFlagsEnabled(env) || results.length === 0) return results;
+
+    const needsUnread = (row: T) => ![true, false, 0, 1].includes(row.unread as boolean | number);
+    const ids = [...new Set(results.filter(needsUnread).map(row => Number(row.id)))]
+        .filter(id => Number.isInteger(id) && id > 0);
+    if (ids.length === 0) {
+        return results.map(row => ({ ...row, unread: Boolean(row.unread) }));
+    }
+
+    const placeholders = ids.map(() => '?').join(',');
+    const { results: flagRows } = await db.prepare(
+        `SELECT mf.mail_id FROM mail_flags mf`
+        + ` JOIN raw_mails rm ON rm.id = mf.mail_id`
+        + ` JOIN address a ON a.id = mf.address_id AND a.name = rm.address`
+        + ` WHERE mf.flag = ? AND mf.mail_id IN (${placeholders})`
+    ).bind(MailFlag.UNREAD, ...ids).all<{ mail_id: number }>();
+    const unreadIds = new Set(flagRows.map(row => Number(row.mail_id)));
+
+    return results.map(row => ({
+        ...row,
+        unread: needsUnread(row) ? unreadIds.has(Number(row.id)) : Boolean(row.unread),
+    }));
+};
+
+export const serializeMailState = async <T extends Record<string, unknown>>(
+    db: D1Database,
    row: T,
    env: Bindings,
-): T => {
-    const result = { ...row };
-    const flags = Number(result.flags ?? 0);
-    delete result.flags;
-    if (!getBooleanValue(env.ENABLE_MAIL_FLAGS)) {
-        return result;
-    }
-    result.unread = (flags & MAIL_FLAGS.UNREAD) !== 0;
+): Promise<T> => {
+    const [result] = await serializeMailStates(db, [row], env);
    return result;
 };

@@ -112,8 +153,8 @@ const resolveInitialMailFlags = async (
    _env: Bindings,
    _address: string,
    _parsedEmailContext: ParsedEmailContext,
-): Promise<number> => {
-    return MAIL_FLAGS.UNREAD;
+): Promise<number[]> => {
+    return [MailFlag.UNREAD];
 };

 export const updateInitialMailFlags = async (
@@ -125,43 +166,62 @@ export const updateInitialMailFlags = async (
    parsedEmailContext: ParsedEmailContext,
 ) => {
    if (!enabled || !Number.isInteger(mailId) || mailId <= 0) return;
+
    const flags = await resolveInitialMailFlags(env, address, parsedEmailContext);
-    await db.prepare(`UPDATE raw_mails SET flags = ? WHERE id = ?`).bind(flags, mailId).run();
+    if (flags.length === 0) return;
+
+    const selects = flags.map(() => 'SELECT ?, id, ? FROM address WHERE name = ?').join(' UNION ALL ');
+    await db.prepare(
+        `INSERT OR IGNORE INTO mail_flags (mail_id, address_id, flag) ${selects}`
+    ).bind(...flags.flatMap(flag => [mailId, flag, address])).run();
 };

-export type MailStateUpdate = {
-    ids: number[];
-    state: string;
-    add: number;
-    remove: number;
-};
-
-export type MailReadStatusQuery = {
-    clause: string;
-    params: string[];
+export type MailStateQuery = {
+    join: string;
+    clause?: string;
+    orderBy?: string;
+    unread?: boolean;
+    params: number[];
 };

 export const getMailStateQuery = (
    value: string | undefined,
-    column: 'flags' | 'rm.flags',
+    mailAlias: string,
+    addressIdColumn: string,
    customStates: MailStateDefinition[] = [],
-): MailReadStatusQuery | undefined | null => {
+): MailStateQuery | undefined | null => {
    if (value === undefined) return undefined;
+
    const definition = getMailStateDefinition(value, customStates);
    if (!definition) return null;
    if (!definition.filter) return undefined;
-    const operator = definition.filter.set ? '!=' : '=';
+
+    const { flag, present } = definition.filter;
+    const joinType = present ? 'JOIN' : 'LEFT JOIN';
    return {
-        clause: `(COALESCE(${column}, 0) & ?) ${operator} 0`,
-        params: [String(definition.filter.mask)],
+        join: ` ${joinType} mail_flags mail_state_flags`
+            + ` ON mail_state_flags.mail_id = ${mailAlias}.id`
+            + ` AND mail_state_flags.address_id = ${addressIdColumn}`
+            + ` AND mail_state_flags.flag = ?`,
+        clause: present ? undefined : 'mail_state_flags.mail_id IS NULL',
+        orderBy: present ? 'mail_state_flags.mail_id desc' : undefined,
+        unread: definition.unread,
+        params: [flag],
    };
 };

+type MailStateUpdate = {
+    ids: number[];
+    mutation: MailStateMutation;
+    unread?: boolean;
+};
+
 const parseMailStateUpdate = (
    value: unknown,
    customStates: MailStateDefinition[] = [],
 ): MailStateUpdate | null => {
    if (!value || typeof value !== 'object') return null;
+
    const body = value as Record<string, unknown>;
    if (!Array.isArray(body.ids) || body.ids.length === 0 || body.ids.length > 100) return null;
    if (body.ids.some(id => typeof id !== 'number')) return null;
@@ -171,24 +231,7 @@ const parseMailStateUpdate = (

    const definition = getMailStateDefinition(body.state, customStates);
    if (!definition?.mutation) return null;
-    return {
-        ids,
-        state: definition.value,
-        add: definition.mutation.add,
-        remove: definition.mutation.remove,
-    };
-};
-
-const getMailStateUpdateExpression = (
-    update: MailStateUpdate,
-    column = 'flags',
-): { expression: string; params: number[]; condition?: string; conditionParams?: number[] } => {
-    return {
-        expression: `((COALESCE(${column}, 0) | ?) & ~?)`,
-        params: [update.add, update.remove],
-        condition: `((COALESCE(${column}, 0) & ?) != ? OR (COALESCE(${column}, 0) & ?) != 0)`,
-        conditionParams: [update.add, update.add, update.remove],
-    };
+    return { ids, mutation: definition.mutation, unread: definition.unread };
 };

 type MailScope = {
@@ -206,27 +249,87 @@ export const applyMailStateUpdate = async (
    const update = parseMailStateUpdate(value, customStates);
    if (!update) return null;

-    const placeholders = update.ids.map(() => '?').join(',');
-    const statusUpdate = getMailStateUpdateExpression(update);
-    const condition = statusUpdate.condition ? ` AND ${statusUpdate.condition}` : '';
-    const result = await db.prepare(
-        `UPDATE raw_mails SET flags = ${statusUpdate.expression}`
-        + ` WHERE id IN (${placeholders}) AND (${scope.clause})${condition}`
-    ).bind(
-        ...statusUpdate.params,
-        ...update.ids,
-        ...scope.params,
-        ...(statusUpdate.conditionParams ?? []),
-    ).run();
-    if (!result.success) return { success: false, changes: 0, results: [] };
+    const idPlaceholders = update.ids.map(() => '?').join(',');
+    const targetWhere = `rm.id IN (${idPlaceholders}) AND (${scope.clause})`;
+    const statements: D1PreparedStatement[] = [];

+    if (update.mutation.remove?.length) {
+        const flagPlaceholders = update.mutation.remove.map(() => '?').join(',');
+        statements.push(db.prepare(
+            `DELETE FROM mail_flags WHERE flag IN (${flagPlaceholders})`
+            + ` AND mail_id IN (`
+            + `SELECT rm.id FROM raw_mails rm JOIN address a ON a.name = rm.address`
+            + ` WHERE ${targetWhere})`
+        ).bind(...update.mutation.remove, ...update.ids, ...scope.params));
+    }
+
+    if (update.mutation.add !== undefined) {
+        statements.push(db.prepare(
+            `INSERT OR IGNORE INTO mail_flags (mail_id, address_id, flag)`
+            + ` SELECT rm.id, a.id, ?`
+            + ` FROM raw_mails rm JOIN address a ON a.name = rm.address`
+            + ` WHERE ${targetWhere}`
+        ).bind(update.mutation.add, ...update.ids, ...scope.params));
+    }
+
+    const mutationResults = await db.batch(statements);
+    if (mutationResults.some(result => !result.success)) {
+        return { success: false, changes: 0, results: [] };
+    }
+
+    const unreadSelect = update.unread === undefined ? '' : `, ${update.unread ? 1 : 0} AS unread`;
    const { results } = await db.prepare(
-        `SELECT id, flags FROM raw_mails`
-        + ` WHERE id IN (${placeholders}) AND (${scope.clause})`
+        `SELECT rm.id${unreadSelect} FROM raw_mails rm JOIN address a ON a.name = rm.address`
+        + ` WHERE ${targetWhere}`
    ).bind(...update.ids, ...scope.params).all();
+
    return {
        success: true,
-        changes: result.meta.changes ?? 0,
-        results: results.map(row => serializeMailState(row, env)),
+        changes: mutationResults.reduce((total, result) => total + (result.meta.changes ?? 0), 0),
+        results: await serializeMailStates(db, results, env),
    };
 };
+
+export const prepareRawMailDeleteStatements = (
+    db: D1Database,
+    env: Bindings,
+    whereClause: string,
+    params: (string | number)[],
+): D1PreparedStatement[] => {
+    const deleteMail = db.prepare(
+        `DELETE FROM raw_mails WHERE ${whereClause}`
+    ).bind(...params);
+    if (!isMailFlagsEnabled(env)) return [deleteMail];
+
+    const deleteFlags = db.prepare(
+        `DELETE FROM mail_flags WHERE mail_id IN (`
+        + `SELECT id FROM raw_mails WHERE ${whereClause})`
+    ).bind(...params);
+    return [deleteFlags, deleteMail];
+};
+
+export const deleteRawMails = async (
+    db: D1Database,
+    env: Bindings,
+    whereClause: string,
+    params: (string | number)[],
+): Promise<D1Result> => {
+    const results = await db.batch(prepareRawMailDeleteStatements(db, env, whereClause, params));
+    return results[results.length - 1];
+};
+
+export const cleanupOrphanMailFlags = async (
+    db: D1Database,
+    env: Bindings,
+    limit = 1000,
+): Promise<number> => {
+    if (!isMailFlagsEnabled(env)) return 0;
+
+    const result = await db.prepare(
+        `DELETE FROM mail_flags WHERE (mail_id, flag) IN (`
+        + `SELECT mf.mail_id, mf.flag FROM mail_flags mf`
+        + ` LEFT JOIN raw_mails rm ON rm.id = mf.mail_id`
+        + ` WHERE rm.id IS NULL LIMIT ?)`
+    ).bind(limit).run();
+    return result.meta.changes ?? 0;
+};