From f12163490e07566f43eb21a5aa16dca59cd9467b Mon Sep 17 00:00:00 2001 From: dreamhunter2333 Date: Thu, 27 Aug 2026 14:03:50 +0800 Subject: [PATCH] refactor: store mail flags in sparse relation table --- CHANGELOG.md | 2 +- CHANGELOG_EN.md | 2 +- db/2026-08-25-mail-flags.sql | 10 +- db/schema.sql | 11 +- e2e/tests/api/mail-flags.spec.ts | 303 +++++++++++++++++- e2e/tests/browser/mail-flags.spec.ts | 89 +++++ frontend/src/components/MailBox.vue | 4 +- frontend/src/views/index/SimpleIndex.vue | 2 +- .../docs/en/guide/feature/mail-api.md | 2 +- vitepress-docs/docs/en/guide/worker-vars.md | 2 +- .../docs/zh/guide/feature/mail-api.md | 2 +- vitepress-docs/docs/zh/guide/worker-vars.md | 2 +- worker/src/admin_api/address_api.ts | 21 +- worker/src/admin_api/admin_mail_api.ts | 9 +- worker/src/admin_api/cleanup_api.ts | 2 + worker/src/admin_api/db_api.ts | 32 +- worker/src/admin_api/e2e_test_api.ts | 16 +- worker/src/admin_api/index.ts | 1 + worker/src/common.ts | 51 +-- worker/src/mail_flags.ts | 269 +++++++++++----- worker/src/mails_api/mails_crud.ts | 54 ++-- worker/src/mails_api/parsed_mail_api.ts | 3 +- worker/src/models/index.ts | 1 - worker/src/telegram_api/miniapp.ts | 4 +- worker/src/user_api/user_mail_api.ts | 35 +- 25 files changed, 750 insertions(+), 179 deletions(-) create mode 100644 e2e/tests/browser/mail-flags.spec.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index ecab97a..1e6ec2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ ### Features -- feat: |邮件状态| 新增可选邮件状态功能，新邮件支持已读/未读、打开自动已读、手动切换状态、本页全部已读及按状态筛选；后端统一返回系统状态枚举与自定义分组，前端直接使用返回值显示和移动邮件 +- feat: |邮件状态| 新增基于独立稀疏关联表的可选邮件状态功能，不修改原邮件表且历史邮件默认已读；新邮件支持已读/未读、打开自动已读、手动切换、本页全部已读及索引化状态筛选，后端统一返回系统状态枚举与自定义分组 - feat: |Admin| 数据库页面新增 D1 存储容量展示，支持选择并保存 Free 或 Workers Paid 套餐，对比当前数据库大小和容量上限 - feat: |用户系统| 用户中心新增发送邮件、与收件箱一致的可按绑定地址过滤的发件箱，以及地址管理凭证弹框；提供使用用户 JWT 的地址设置、发信权限申请、发信及发件箱 API diff --git a/CHANGELOG_EN.md b/CHANGELOG_EN.md index 5fcd551..d33c6a7 100644 --- a/CHANGELOG_EN.md +++ b/CHANGELOG_EN.md @@ -10,7 +10,7 @@ ### Features -- feat: |Mail State| Add optional read state for new mail, automatic read-on-open, manual state toggling, mark-current-page-read and state filters; the backend returns the combined system-state enum and custom groups for clients to render and use as move targets +- feat: |Mail State| Add optional mail states backed by a separate sparse relation table, leaving the raw-mail table unchanged and treating historical mail as read; support automatic read-on-open, manual toggling, mark-current-page-read, indexed state filters, and backend-owned system and custom state definitions - feat: |Admin| Add D1 storage capacity details to the database page, with persistent Free and Workers Paid plan selection and a comparison between the current database size and capacity limit - feat: |User| Add mail composition, inbox-style sent-item filtering by bound address, and the shared address-credentials dialog to the user center, backed by User JWT APIs for address settings, send-access requests, sending, and sent-item management diff --git a/db/2026-08-25-mail-flags.sql b/db/2026-08-25-mail-flags.sql index 7fcff9d..8385049 100644 --- a/db/2026-08-25-mail-flags.sql +++ b/db/2026-08-25-mail-flags.sql @@ -1 +1,9 @@ -ALTER TABLE raw_mails ADD COLUMN flags INTEGER; +CREATE TABLE IF NOT EXISTS mail_flags ( + mail_id INTEGER NOT NULL, + address_id INTEGER NOT NULL, + flag INTEGER NOT NULL, + PRIMARY KEY (mail_id, flag) +) WITHOUT ROWID; + +CREATE INDEX IF NOT EXISTS idx_mail_flags_address_flag_mail +ON mail_flags(address_id, flag, mail_id DESC); diff --git a/db/schema.sql b/db/schema.sql index a9a8523..fb897a0 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -6,7 +6,6 @@ CREATE TABLE IF NOT EXISTS raw_mails ( raw TEXT, raw_blob BLOB, metadata TEXT, - flags INTEGER, created_at DATETIME DEFAULT CURRENT_TIMESTAMP ); @@ -16,6 +15,16 @@ CREATE INDEX IF NOT EXISTS idx_raw_mails_created_at ON raw_mails(created_at); CREATE INDEX IF NOT EXISTS idx_raw_mails_message_id ON raw_mails(message_id); +CREATE TABLE IF NOT EXISTS mail_flags ( + mail_id INTEGER NOT NULL, + address_id INTEGER NOT NULL, + flag INTEGER NOT NULL, + PRIMARY KEY (mail_id, flag) +) WITHOUT ROWID; + +CREATE INDEX IF NOT EXISTS idx_mail_flags_address_flag_mail +ON mail_flags(address_id, flag, mail_id DESC); + CREATE TABLE IF NOT EXISTS address ( id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT UNIQUE, diff --git a/e2e/tests/api/mail-flags.spec.ts b/e2e/tests/api/mail-flags.spec.ts index 8680d23..9266f6d 100644 --- a/e2e/tests/api/mail-flags.spec.ts +++ b/e2e/tests/api/mail-flags.spec.ts @@ -1,11 +1,49 @@ -import { test, expect } from '@playwright/test'; +import { test, expect, type APIRequestContext } from '@playwright/test'; import { WORKER_URL, + WORKER_URL_ENV_OFF, createTestAddress, deleteAddress, + hashPassword, seedTestMail, } from '../../fixtures/test-helpers'; +const addressHeaders = (jwt: string) => ({ Authorization: `Bearer ${jwt}` }); + +async function listAddressMails( + request: APIRequestContext, + jwt: string, + state = 'all', + baseUrl = WORKER_URL, +) { + const response = await request.get( + `${baseUrl}/api/mails?limit=100&offset=0&mail_state=${state}`, + { headers: addressHeaders(jwt) }, + ); + expect(response.ok()).toBe(true); + return await response.json(); +} + +async function updateAddressMailState( + request: APIRequestContext, + jwt: string, + ids: number[], + state: string, +) { + const response = await request.patch(`${WORKER_URL}/api/mails/state`, { + headers: addressHeaders(jwt), + data: { ids, state }, + }); + expect(response.ok()).toBe(true); + return await response.json(); +} + +async function getStoredFlags(request: APIRequestContext, mailId: number) { + const response = await request.get(`${WORKER_URL}/admin/test/mail_flags?mail_id=${mailId}`); + expect(response.ok()).toBe(true); + return (await response.json()).results as { mail_id: number; address_id: number; flag: number }[]; +} + test.describe('Mail Read Status', () => { test('new mail is unread and can be marked as read without changing other mailboxes', async ({ request }) => { const first = await createTestAddress(request, 'mail-flags-first'); @@ -28,6 +66,23 @@ test.describe('Mail Read Status', () => { expect(results).toHaveLength(1); expect(results[0].flags).toBeUndefined(); expect(results[0].unread).toBe(true); + expect(await getStoredFlags(request, results[0].id)).toEqual([{ + mail_id: results[0].id, + address_id: first.address_id, + flag: 0, + }]); + + const detailRes = await request.get(`${WORKER_URL}/api/mails/${results[0].id}`, { + headers: addressHeaders(first.jwt), + }); + expect(detailRes.ok()).toBe(true); + expect((await detailRes.json()).unread).toBe(true); + + const parsedDetailRes = await request.get(`${WORKER_URL}/api/parsed_mail/${results[0].id}`, { + headers: addressHeaders(first.jwt), + }); + expect(parsedDetailRes.ok()).toBe(true); + expect((await parsedDetailRes.json()).unread).toBe(true); const unreadRes = await request.get( `${WORKER_URL}/api/mails?limit=10&offset=0&mail_state=unread`, @@ -50,6 +105,7 @@ test.describe('Mail Read Status', () => { const updateResult = await updateRes.json(); expect(updateResult.changes).toBe(1); expect(updateResult.results[0].unread).toBe(false); + expect(await getStoredFlags(request, results[0].id)).toEqual([]); const updatedListRes = await request.get(`${WORKER_URL}/api/mails?limit=10&offset=0`, { headers: { Authorization: `Bearer ${first.jwt}` }, @@ -68,18 +124,68 @@ test.describe('Mail Read Status', () => { }); expect(unreadStateRes.ok()).toBe(true); expect((await unreadStateRes.json()).results[0].unread).toBe(true); + expect(await getStoredFlags(request, results[0].id)).toHaveLength(1); } finally { await deleteAddress(request, first.jwt); await deleteAddress(request, second.jwt); } }); + test('supports batch and idempotent updates with accurate filters and cleanup', async ({ request }) => { + const mailbox = await createTestAddress(request, 'mail-flags-batch'); + try { + for (let index = 0; index < 3; index += 1) { + await seedTestMail(request, mailbox.address, { subject: `Flag batch ${index}` }); + } + + const initial = await listAddressMails(request, mailbox.jwt); + const ids = initial.results.map((mail: { id: number }) => mail.id); + expect(initial.count).toBe(3); + expect(initial.results.every((mail: { unread: boolean }) => mail.unread)).toBe(true); + + const firstUpdate = await updateAddressMailState(request, mailbox.jwt, ids.slice(0, 2), 'read'); + expect(firstUpdate.changes).toBe(2); + expect(firstUpdate.results).toHaveLength(2); + expect(firstUpdate.results.every((mail: { unread: boolean }) => !mail.unread)).toBe(true); + + const duplicateUpdate = await updateAddressMailState(request, mailbox.jwt, ids.slice(0, 2), 'read'); + expect(duplicateUpdate.changes).toBe(0); + + const read = await listAddressMails(request, mailbox.jwt, 'read'); + expect(read.count).toBe(2); + expect(read.results.map((mail: { id: number }) => mail.id).sort()).toEqual(ids.slice(0, 2).sort()); + expect(read.results.every((mail: { unread: boolean }) => !mail.unread)).toBe(true); + + const unread = await listAddressMails(request, mailbox.jwt, 'unread'); + expect(unread.count).toBe(1); + expect(unread.results[0].id).toBe(ids[2]); + expect(unread.results[0].unread).toBe(true); + + const duplicateUnread = await updateAddressMailState(request, mailbox.jwt, [ids[2]], 'unread'); + expect(duplicateUnread.changes).toBe(0); + + const deleteRes = await request.delete(`${WORKER_URL}/api/mails/${ids[2]}`, { + headers: addressHeaders(mailbox.jwt), + }); + expect(deleteRes.ok()).toBe(true); + expect(await getStoredFlags(request, ids[2])).toEqual([]); + expect((await updateAddressMailState(request, mailbox.jwt, [ids[2]], 'unread')).changes).toBe(0); + } finally { + await deleteAddress(request, mailbox.jwt); + } + }); + test('rejects unsupported mail states', async ({ request }) => { const { jwt } = await createTestAddress(request, 'mail-flags-invalid'); try { for (const data of [ { ids: [1], state: 'invalid' }, { ids: [1] }, + { ids: [], state: 'read' }, + { ids: [0], state: 'read' }, + { ids: [1.5], state: 'read' }, + { ids: ['1'], state: 'read' }, + { ids: Array.from({ length: 101 }, (_, index) => index + 1), state: 'read' }, ]) { const res = await request.patch(`${WORKER_URL}/api/mails/state`, { headers: { Authorization: `Bearer ${jwt}` }, @@ -91,4 +197,199 @@ test.describe('Mail Read Status', () => { await deleteAddress(request, jwt); } }); + + test('mail without a flag relation is read by default', async ({ request }) => { + const mailbox = await createTestAddress(request, 'mail-flags-history'); + try { + const seedRes = await request.post(`${WORKER_URL}/admin/test/seed_mail`, { + data: { + address: mailbox.address, + raw: [ + `From: sender@example.com`, + `To: ${mailbox.address}`, + `Subject: Historical mail`, + `Message-ID: `, + ``, + `Historical body`, + ].join('\r\n'), + }, + }); + expect(seedRes.ok()).toBe(true); + + const list = async (state: string) => { + const response = await request.get( + `${WORKER_URL}/api/mails?limit=10&offset=0&mail_state=${state}`, + { headers: { Authorization: `Bearer ${mailbox.jwt}` } }, + ); + expect(response.ok()).toBe(true); + return await response.json(); + }; + + const historical = (await list('all')).results[0]; + expect(historical.unread).toBe(false); + expect(await getStoredFlags(request, historical.id)).toEqual([]); + expect((await list('read')).results).toHaveLength(1); + expect((await list('unread')).results).toHaveLength(0); + } finally { + await deleteAddress(request, mailbox.jwt); + } + }); + + test('user APIs query and mutate flags only for bound addresses', async ({ request }) => { + const addresses: Awaited>[] = []; + let outsider: Awaited> | undefined; + let originalSettings: Record | undefined; + let userId: number | undefined; + + try { + const settingsRes = await request.get(`${WORKER_URL}/admin/user_settings`); + expect(settingsRes.ok()).toBe(true); + originalSettings = await settingsRes.json(); + const enableRes = await request.post(`${WORKER_URL}/admin/user_settings`, { + data: { ...originalSettings, enable: true, enableMailVerify: false, maxAddressCount: 0 }, + }); + expect(enableRes.ok()).toBe(true); + + const email = `mail-flags-user-${Date.now()}@test.example.com`; + const password = hashPassword('mail-flags-password'); + const registerRes = await request.post(`${WORKER_URL}/user_api/register`, { + data: { email, password }, + }); + expect(registerRes.ok()).toBe(true); + const loginRes = await request.post(`${WORKER_URL}/user_api/login`, { + data: { email, password }, + }); + expect(loginRes.ok()).toBe(true); + const { jwt: userJwt } = await loginRes.json(); + const payload = JSON.parse(Buffer.from(userJwt.split('.')[1], 'base64url').toString('utf8')); + userId = payload.user_id; + + addresses.push( + await createTestAddress(request, 'mail-flags-user-a'), + await createTestAddress(request, 'mail-flags-user-b'), + ); + outsider = await createTestAddress(request, 'mail-flags-user-outsider'); + + for (const mailbox of addresses) { + const bindRes = await request.post(`${WORKER_URL}/user_api/bind_address`, { + headers: { + ...addressHeaders(mailbox.jwt), + 'x-user-token': userJwt, + }, + }); + expect(bindRes.ok()).toBe(true); + await seedTestMail(request, mailbox.address, { subject: `Bound ${mailbox.address}` }); + } + await seedTestMail(request, outsider.address, { subject: 'Outsider unread' }); + + const statesRes = await request.get(`${WORKER_URL}/user_api/mail-states`, { + headers: { 'x-user-token': userJwt }, + }); + expect(statesRes.ok()).toBe(true); + expect((await statesRes.json()).results.map((state: { value: string }) => state.value)) + .toEqual(['all', 'unread', 'read']); + + const userList = async (state: string, address?: string) => { + const addressQuery = address ? `&address=${encodeURIComponent(address)}` : ''; + const response = await request.get( + `${WORKER_URL}/user_api/mails?limit=20&offset=0&mail_state=${state}${addressQuery}`, + { headers: { 'x-user-token': userJwt } }, + ); + expect(response.ok()).toBe(true); + return await response.json(); + }; + + const unread = await userList('unread'); + expect(unread.count).toBe(2); + expect(new Set(unread.results.map((mail: { address: string }) => mail.address))) + .toEqual(new Set(addresses.map(mailbox => mailbox.address))); + expect((await userList('unread', addresses[0].address)).results).toHaveLength(1); + expect((await userList('unread', outsider.address)).results).toHaveLength(0); + + const firstMail = unread.results.find( + (mail: { address: string }) => mail.address === addresses[0].address, + ); + const updateRes = await request.patch(`${WORKER_URL}/user_api/mails/state`, { + headers: { 'x-user-token': userJwt }, + data: { ids: [firstMail.id], state: 'read' }, + }); + expect(updateRes.ok()).toBe(true); + expect((await updateRes.json()).results[0].unread).toBe(false); + expect((await userList('read')).results.map((mail: { id: number }) => mail.id)) + .toContain(firstMail.id); + + const outsiderMail = (await listAddressMails(request, outsider.jwt)).results[0]; + const deniedRes = await request.patch(`${WORKER_URL}/user_api/mails/state`, { + headers: { 'x-user-token': userJwt }, + data: { ids: [outsiderMail.id], state: 'read' }, + }); + expect(deniedRes.ok()).toBe(true); + expect((await deniedRes.json()).changes).toBe(0); + expect((await listAddressMails(request, outsider.jwt, 'unread')).results).toHaveLength(1); + } finally { + await Promise.allSettled( + [...addresses, outsider].filter((mailbox): mailbox is NonNullable => mailbox !== undefined) + .map((mailbox) => deleteAddress(request, mailbox.jwt)), + ); + if (userId !== undefined) { + await request.delete(`${WORKER_URL}/admin/users/${userId}`); + } + if (originalSettings) { + await request.post(`${WORKER_URL}/admin/user_settings`, { data: originalSettings }); + } + } + }); + + test('disabled feature keeps legacy mail responses and rejects state APIs', async ({ request }) => { + test.skip(!WORKER_URL_ENV_OFF, 'WORKER_URL_ENV_OFF is not configured'); + + const createRes = await request.post(`${WORKER_URL_ENV_OFF}/api/new_address`, { + data: { name: `mail-flags-off-${Date.now()}`, domain: 'test.example.com' }, + }); + expect(createRes.ok()).toBe(true); + const mailbox = await createRes.json(); + + try { + const raw = [ + `From: sender@example.com`, + `To: ${mailbox.address}`, + `Subject: Flags disabled`, + `Message-ID: `, + ``, + `Disabled body`, + ].join('\r\n'); + const receiveRes = await request.post(`${WORKER_URL_ENV_OFF}/admin/test/receive_mail`, { + data: { from: 'sender@example.com', to: mailbox.address, raw }, + }); + expect(receiveRes.ok()).toBe(true); + + const listRes = await request.get(`${WORKER_URL_ENV_OFF}/api/mails?limit=10&offset=0`, { + headers: addressHeaders(mailbox.jwt), + }); + expect(listRes.ok()).toBe(true); + const list = await listRes.json(); + expect(list.results).toHaveLength(1); + expect(list.results[0]).not.toHaveProperty('unread'); + expect(list.results[0]).not.toHaveProperty('flags'); + + const statesRes = await request.get(`${WORKER_URL_ENV_OFF}/api/mail-states`, { + headers: addressHeaders(mailbox.jwt), + }); + expect(statesRes.status()).toBe(403); + + const filterRes = await request.get( + `${WORKER_URL_ENV_OFF}/api/mails?limit=10&offset=0&mail_state=unread`, + { headers: addressHeaders(mailbox.jwt) }, + ); + expect(filterRes.status()).toBe(403); + + const updateRes = await request.patch(`${WORKER_URL_ENV_OFF}/api/mails/state`, { + headers: addressHeaders(mailbox.jwt), + data: { ids: [list.results[0].id], state: 'read' }, + }); + expect(updateRes.status()).toBe(403); + } finally { + await request.delete(`${WORKER_URL_ENV_OFF}/admin/delete_address/${mailbox.address_id}`); + } + }); }); diff --git a/e2e/tests/browser/mail-flags.spec.ts b/e2e/tests/browser/mail-flags.spec.ts new file mode 100644 index 0000000..8a592f7 --- /dev/null +++ b/e2e/tests/browser/mail-flags.spec.ts @@ -0,0 +1,89 @@ +import { expect, request as apiRequest, test } from '@playwright/test'; + +import { + FRONTEND_URL, + WORKER_URL, + createTestAddress, + deleteAddress, + seedTestMail, +} from '../../fixtures/test-helpers'; + +test.describe('Mail state browser flow', () => { + test('opens, toggles, filters and marks the current page read', async ({ page }) => { + const request = await apiRequest.newContext(); + let jwt: string | undefined; + + try { + const mailbox = await createTestAddress(request, 'mail-flags-browser'); + jwt = mailbox.jwt; + const subjects = [`Unread A ${Date.now()}`, `Unread B ${Date.now()}`]; + for (const subject of subjects) { + await seedTestMail(request, mailbox.address, { subject }); + } + + await page.goto(`${FRONTEND_URL}/en/`); + await page.evaluate(() => localStorage.setItem('mailListView', 'true')); + await page.goto(`${FRONTEND_URL}/en/?jwt=${jwt}`); + + for (const subject of subjects) { + await expect(page.getByText(subject, { exact: true })).toBeVisible({ timeout: 10_000 }); + } + await expect(page.getByText('Unread', { exact: true })).toHaveCount(2); + + const openStateResponse = page.waitForResponse((response) => { + return new URL(response.url()).pathname === '/api/mails/state' + && response.request().method() === 'PATCH'; + }); + await page.getByText(subjects[0], { exact: true }).click(); + expect((await openStateResponse).ok()).toBe(true); + await expect(page.getByRole('button', { name: 'Mark as Unread' })).toBeVisible(); + + const unreadAfterOpen = await request.get( + `${WORKER_URL}/api/mails?limit=10&offset=0&mail_state=unread`, + { headers: { Authorization: `Bearer ${jwt}` } }, + ); + expect((await unreadAfterOpen.json()).results).toHaveLength(1); + + const toggleResponse = page.waitForResponse((response) => { + return new URL(response.url()).pathname === '/api/mails/state' + && response.request().method() === 'PATCH'; + }); + await page.getByRole('button', { name: 'Mark as Unread' }).click(); + expect((await toggleResponse).ok()).toBe(true); + await expect(page.getByRole('button', { name: 'Mark as Read' })).toBeVisible(); + + await page.getByRole('button', { name: 'Back to List' }).click(); + const pageReadResponse = page.waitForResponse((response) => { + if (new URL(response.url()).pathname !== '/api/mails/state') return false; + if (response.request().method() !== 'PATCH') return false; + const body = response.request().postDataJSON(); + return body.state === 'read' && body.ids.length === 2; + }); + await page.getByRole('button', { name: 'Mark This Page as Read' }).click(); + expect((await pageReadResponse).ok()).toBe(true); + await expect(page.getByRole('button', { name: 'Mark This Page as Read' })).toBeHidden(); + + const unreadAfterPage = await request.get( + `${WORKER_URL}/api/mails?limit=10&offset=0&mail_state=unread`, + { headers: { Authorization: `Bearer ${jwt}` } }, + ); + expect((await unreadAfterPage.json()).results).toHaveLength(0); + + const unreadFilterResponse = page.waitForResponse((response) => { + const url = new URL(response.url()); + return url.pathname === '/api/mails' && url.searchParams.get('mail_state') === 'unread'; + }); + const stateSelect = page.locator('.n-select').filter({ hasText: 'All Mail' }).first(); + await stateSelect.click(); + await page.locator('.n-base-select-option').filter({ hasText: /^Unread$/ }).click(); + expect((await unreadFilterResponse).ok()).toBe(true); + await expect(page.getByText('Your inbox is empty')).toBeVisible(); + } finally { + try { + if (jwt) await deleteAddress(request, jwt); + } finally { + await request.dispose(); + } + } + }); +}); diff --git a/frontend/src/components/MailBox.vue b/frontend/src/components/MailBox.vue index 97c9e7e..b85640f 100644 --- a/frontend/src/components/MailBox.vue +++ b/frontend/src/components/MailBox.vue @@ -142,7 +142,9 @@ const updateUnreadState = async (mails, state) => { } } -const markMailsRead = async (mails) => updateUnreadState(mails, getReadStateValue(false)) +const markMailsRead = async (mails) => { + return await updateUnreadState(mails.filter(isMailUnread), getReadStateValue(false)) +} const toggleCurrentMailUnread = async () => { if (!curMail.value) return diff --git a/frontend/src/views/index/SimpleIndex.vue b/frontend/src/views/index/SimpleIndex.vue index 69efc67..6f8d4b0 100644 --- a/frontend/src/views/index/SimpleIndex.vue +++ b/frontend/src/views/index/SimpleIndex.vue @@ -55,7 +55,7 @@ const fetchMails = async () => { totalCount.value = count > 0 ? count : totalCount.value; const rawMail = results && results.length > 0 ? results[0] : null currentMail.value = rawMail ? await processItem(rawMail) : null - if (openSettings.value.enableMailStates && rawMail) { + if (openSettings.value.enableMailStates && rawMail?.unread) { const state = getReadStateValue(false) if (!state) return const response = await api.fetch(`/api/mails/state`, { diff --git a/vitepress-docs/docs/en/guide/feature/mail-api.md b/vitepress-docs/docs/en/guide/feature/mail-api.md index 6a0bfb4..31028c0 100644 --- a/vitepress-docs/docs/en/guide/feature/mail-api.md +++ b/vitepress-docs/docs/en/guide/feature/mail-api.md @@ -21,7 +21,7 @@ res = requests.get( ## Mail State API -After enabling `ENABLE_MAIL_FLAGS` and running the database migration, each mail response includes the boolean field `unread`. The backend owns storage, historical `NULL` compatibility, and state calculation. +After enabling `ENABLE_MAIL_FLAGS` and running the database migration, each mail response includes the boolean field `unread`. Mail states live in a separate sparse relation table without changing `raw_mails`; historical mail without a state record is read by default, and the backend owns all state calculation. With an Address JWT, use `GET /api/mail-states` to retrieve the currently available system states. This endpoint is the extension point for future address-specific custom states. The frontend uses each returned `value` directly for filtering and moving, and displays its `label_key` or `label`. diff --git a/vitepress-docs/docs/en/guide/worker-vars.md b/vitepress-docs/docs/en/guide/worker-vars.md index 8f59349..2861838 100644 --- a/vitepress-docs/docs/en/guide/worker-vars.md +++ b/vitepress-docs/docs/en/guide/worker-vars.md @@ -102,7 +102,7 @@ | `REMOVE_EXCEED_SIZE_ATTACHMENT` | Text/JSON | If attachment exceeds 2MB, remove it, email may lose some information due to parsing | `true` | | `REMOVE_ALL_ATTACHMENT` | Text/JSON | Remove all attachments, email may lose some information due to parsing | `true` | | `ENABLE_MAIL_GZIP` | Text/JSON | When enabled, new emails are gzip-compressed and stored in `raw_blob` column to save D1 database space. Existing plaintext `raw` data is automatically compatible for reading. **Run database migration first (`Admin -> Quick Setup -> Database -> Migrate Database` or `POST /admin/db_migration`) to ensure the `raw_blob` column exists before enabling. This feature adds compression/decompression CPU overhead, so enabling it on a paid Cloudflare Worker plan is recommended.** | `true` | -| `ENABLE_MAIL_FLAGS` | Text/JSON | Enables per-message state in the web inbox. New mail starts unread and can be opened to mark read, toggled manually, filtered by state, or marked read for the current page. Historical `NULL`/`0` values are treated as read. **Run the database migration first so `raw_mails.flags` exists before enabling this option.** | `true` | +| `ENABLE_MAIL_FLAGS` | Text/JSON | Enables per-message state in the web inbox. New mail starts unread and can be opened to mark read, toggled manually, filtered by state, or marked read for the current page. State is stored in a separate sparse relation table without changing `raw_mails`; historical mail without a relation is read by default. **Run the database migration first so the `mail_flags` table exists.** | `true` | | `CLEANUP_BATCH_SIZE` | Number | Per-run limit for mail, sent-mail, and creation/activity-based address cleanup. Defaults to `3000`, valid range `1-5000`. Smaller values reduce per-run D1 pressure; larger values clear backlogs faster | `3000` | > [!NOTE] diff --git a/vitepress-docs/docs/zh/guide/feature/mail-api.md b/vitepress-docs/docs/zh/guide/feature/mail-api.md index 28aebb7..3926868 100644 --- a/vitepress-docs/docs/zh/guide/feature/mail-api.md +++ b/vitepress-docs/docs/zh/guide/feature/mail-api.md @@ -21,7 +21,7 @@ res = requests.get( ## 邮件状态 API -启用 `ENABLE_MAIL_FLAGS` 并完成数据库迁移后，邮件响应会包含布尔字段 `unread`。数据库存储、历史 `NULL` 兼容和状态计算全部由后端处理。 +启用 `ENABLE_MAIL_FLAGS` 并完成数据库迁移后，邮件响应会包含布尔字段 `unread`。邮件状态保存在独立的稀疏关联表中，不修改 `raw_mails`；没有状态记录的历史邮件默认已读，状态计算全部由后端处理。 地址 JWT 使用 `GET /api/mail-states` 获取当前可用的系统状态；该接口也是未来扩展地址自定义状态的入口。前端直接使用其中的 `value` 作为筛选和移动参数，并使用 `label_key` 或 `label` 显示名称。 diff --git a/vitepress-docs/docs/zh/guide/worker-vars.md b/vitepress-docs/docs/zh/guide/worker-vars.md index 7cd4cfa..37054b8 100644 --- a/vitepress-docs/docs/zh/guide/worker-vars.md +++ b/vitepress-docs/docs/zh/guide/worker-vars.md @@ -97,7 +97,7 @@ | `REMOVE_EXCEED_SIZE_ATTACHMENT` | 文本/JSON | 如果附件大小超过 2MB，则删除附件，邮件可能由于解析而丢失一些信息 | `true` | | `REMOVE_ALL_ATTACHMENT` | 文本/JSON | 移除所有附件，邮件可能由于解析而丢失一些信息 | `true` | | `ENABLE_MAIL_GZIP` | 文本/JSON | 启用后新邮件将 Gzip 压缩存储到 `raw_blob` 字段，可节省 D1 数据库空间。已有明文 `raw` 数据自动兼容读取。**启用前请先执行数据库迁移（`Admin -> 快速设置 -> 数据库 -> 升级数据库 Schema` 或 `POST /admin/db_migration`），确保 `raw_blob` 列已创建。该功能会增加压缩/解压 CPU 开销，建议使用 Cloudflare Worker 付费 Plan 再开启。** | `true` | -| `ENABLE_MAIL_FLAGS` | 文本/JSON | 启用网页邮件状态功能。新邮件默认未读，支持打开自动已读、手动切换、按状态筛选及本页全部已读。历史邮件的 `NULL`/`0` 状态默认按已读处理。**启用前必须先执行数据库迁移，确保 `raw_mails.flags` 列已创建。** | `true` | +| `ENABLE_MAIL_FLAGS` | 文本/JSON | 启用网页邮件状态功能。新邮件默认未读，支持打开自动已读、手动切换、按状态筛选及本页全部已读。状态存储在独立的稀疏关联表中，不修改 `raw_mails`，无关联记录的历史邮件默认已读。**启用前必须先执行数据库迁移，确保 `mail_flags` 表已创建。** | `true` | | `CLEANUP_BATCH_SIZE` | 数字 | 邮件、发件箱及按创建/活跃时间清理地址时的单次处理上限，默认 `3000`，有效范围 `1-5000`。较小值可降低单次 D1 压力，较大值可加快积压数据清理 | `3000` | > [!NOTE] diff --git a/worker/src/admin_api/address_api.ts b/worker/src/admin_api/address_api.ts index ba29dee..663a8ef 100644 --- a/worker/src/admin_api/address_api.ts +++ b/worker/src/admin_api/address_api.ts @@ -4,6 +4,7 @@ import { Jwt } from 'hono/utils/jwt' import i18n from '../i18n' import { getBooleanValue } from '../utils' import { newAddress, handleListQuery } from '../common' +import { deleteRawMails, prepareRawMailDeleteStatements } from '../mail_flags' const listAddresses = async (c: Context) => { const { limit, offset, query, sort_by, sort_order } = c.req.query(); @@ -74,10 +75,12 @@ const deleteAddress = async (c: Context) => { // deleted first and the address row last, so the name subqueries still // resolve and a failed statement rolls back the whole deletion const results = await c.env.DB.batch([ - c.env.DB.prepare( - `DELETE FROM raw_mails WHERE address IN` - + ` (select name from address where id = ?) ` - ).bind(id), + ...prepareRawMailDeleteStatements( + c.env.DB, + c.env, + `address IN (select name from address where id = ?)`, + [id], + ), c.env.DB.prepare( `DELETE FROM address_sender WHERE address IN` + ` (select name from address where id = ?) ` @@ -107,10 +110,12 @@ const deleteAddress = async (c: Context) => { const clearInbox = async (c: Context) => { const msgs = i18n.getMessagesbyContext(c); const { id } = c.req.param(); - const { success: mailSuccess } = await c.env.DB.prepare( - `DELETE FROM raw_mails WHERE address IN` - + ` (select name from address where id = ?) ` - ).bind(id).run(); + const { success: mailSuccess } = await deleteRawMails( + c.env.DB, + c.env, + `address IN (select name from address where id = ?)`, + [id], + ); if (!mailSuccess) { return c.text(msgs.OperationFailedMsg, 500) } diff --git a/worker/src/admin_api/admin_mail_api.ts b/worker/src/admin_api/admin_mail_api.ts index 7516b43..0469ecc 100644 --- a/worker/src/admin_api/admin_mail_api.ts +++ b/worker/src/admin_api/admin_mail_api.ts @@ -1,7 +1,7 @@ import { Context } from "hono"; import { handleMailListQuery } from "../common"; import { resolveRawEmailRow } from "../gzip"; -import { serializeMailState } from "../mail_flags"; +import { deleteRawMails, serializeMailState } from "../mail_flags"; export default { getMails: async (c: Context) => { @@ -32,16 +32,15 @@ export default { `SELECT * FROM raw_mails WHERE id = ?` ).bind(id).first(); if (!result) return c.json(null); - return c.json(serializeMailState( + return c.json(await serializeMailState( + c.env.DB, await resolveRawEmailRow(result), c.env, )); }, deleteMail: async (c: Context) => { const { id } = c.req.param(); - const { success } = await c.env.DB.prepare( - `DELETE FROM raw_mails WHERE id = ? ` - ).bind(id).run(); + const { success } = await deleteRawMails(c.env.DB, c.env, `id = ?`, [id]); return c.json({ success: success }) diff --git a/worker/src/admin_api/cleanup_api.ts b/worker/src/admin_api/cleanup_api.ts index ca19989..9535d9a 100644 --- a/worker/src/admin_api/cleanup_api.ts +++ b/worker/src/admin_api/cleanup_api.ts @@ -6,6 +6,7 @@ import { getJsonSetting, saveSetting } from '../utils'; import { CleanupSettings, CustomSqlCleanup } from '../models'; import i18n from '../i18n'; import { LocaleMessages } from '../i18n/type'; +import { cleanupOrphanMailFlags } from '../mail_flags'; // SQL validation error types type SqlValidationError = 'empty' | 'too_long' | 'not_delete' | 'multiple_statements' | 'has_comments'; @@ -84,6 +85,7 @@ export const executeCustomSqlCleanup = async ( console.log(`Executing custom SQL cleanup [${customSql.name}]: ${sql}`); const result = await c.env.DB.prepare(sql).run(); const rowsAffected = result.meta?.changes ?? 0; + await cleanupOrphanMailFlags(c.env.DB, c.env); console.log(`Custom SQL cleanup [${customSql.name}] completed, rows affected: ${rowsAffected}`); return { success: true, rowsAffected }; } catch (error) { diff --git a/worker/src/admin_api/db_api.ts b/worker/src/admin_api/db_api.ts index 38e09fe..5cf21a0 100644 --- a/worker/src/admin_api/db_api.ts +++ b/worker/src/admin_api/db_api.ts @@ -11,7 +11,6 @@ CREATE TABLE IF NOT EXISTS raw_mails ( raw TEXT, raw_blob BLOB, metadata TEXT, - flags INTEGER, created_at DATETIME DEFAULT CURRENT_TIMESTAMP ); @@ -21,6 +20,16 @@ CREATE INDEX IF NOT EXISTS idx_raw_mails_created_at ON raw_mails(created_at); CREATE INDEX IF NOT EXISTS idx_raw_mails_message_id ON raw_mails(message_id); +CREATE TABLE IF NOT EXISTS mail_flags ( + mail_id INTEGER NOT NULL, + address_id INTEGER NOT NULL, + flag INTEGER NOT NULL, + PRIMARY KEY (mail_id, flag) +) WITHOUT ROWID; + +CREATE INDEX IF NOT EXISTS idx_mail_flags_address_flag_mail +ON mail_flags(address_id, flag, mail_id DESC); + CREATE TABLE IF NOT EXISTS address ( id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT UNIQUE, @@ -198,16 +207,17 @@ export default { await c.env.DB.exec(`ALTER TABLE raw_mails ADD COLUMN raw_blob BLOB;`); } } - if (version && version <= "v0.0.7") { - const tableInfo = await c.env.DB.prepare( - `PRAGMA table_info(raw_mails)` - ).all(); - const hasFlags = tableInfo.results?.some( - (col: any) => col.name === 'flags' - ); - if (!hasFlags) { - await c.env.DB.exec(`ALTER TABLE raw_mails ADD COLUMN flags INTEGER;`); - } + if (version && version <= "v0.0.8") { + await c.env.DB.exec(` + CREATE TABLE IF NOT EXISTS mail_flags ( + mail_id INTEGER NOT NULL, + address_id INTEGER NOT NULL, + flag INTEGER NOT NULL, + PRIMARY KEY (mail_id, flag) + ) WITHOUT ROWID; + CREATE INDEX IF NOT EXISTS idx_mail_flags_address_flag_mail + ON mail_flags(address_id, flag, mail_id DESC); + `); } if (version != CONSTANTS.DB_VERSION) { // remove all \r and \n characters from the query string diff --git a/worker/src/admin_api/e2e_test_api.ts b/worker/src/admin_api/e2e_test_api.ts index 36ca7e9..a99df46 100644 --- a/worker/src/admin_api/e2e_test_api.ts +++ b/worker/src/admin_api/e2e_test_api.ts @@ -78,4 +78,18 @@ const receiveMail = async (c: Context) => { }); }; -export default { seedMail, receiveMail }; +const getMailFlags = async (c: Context) => { + if (!getBooleanValue(c.env.E2E_TEST_MODE)) { + return c.text("Not available", 404); + } + const mailId = Number(c.req.query('mail_id')); + if (!Number.isInteger(mailId) || mailId <= 0) { + return c.text("Invalid mail_id", 400); + } + const { results } = await c.env.DB.prepare( + `SELECT mail_id, address_id, flag FROM mail_flags WHERE mail_id = ? ORDER BY flag` + ).bind(mailId).all(); + return c.json({ results }); +}; + +export default { seedMail, receiveMail, getMailFlags }; diff --git a/worker/src/admin_api/index.ts b/worker/src/admin_api/index.ts index 30d9a06..2491742 100644 --- a/worker/src/admin_api/index.ts +++ b/worker/src/admin_api/index.ts @@ -112,3 +112,4 @@ api.post('/admin/ai_extract/settings', ai_extract_settings.saveAiExtractSettings // E2E test endpoints api.post('/admin/test/seed_mail', e2e_test_api.seedMail) api.post('/admin/test/receive_mail', e2e_test_api.receiveMail) +api.get('/admin/test/mail_flags', e2e_test_api.getMailFlags) diff --git a/worker/src/common.ts b/worker/src/common.ts index 341a6fa..96be358 100644 --- a/worker/src/common.ts +++ b/worker/src/common.ts @@ -7,7 +7,7 @@ import { unbindTelegramByAddress } from './telegram_api/common'; import { CONSTANTS } from './constants'; import { AddressCreationSettings, AdminWebhookSettings, ExtractResult, WebhookMail, WebhookSettings } from './models'; import i18n from './i18n'; -import { serializeMailState } from './mail_flags'; +import { deleteRawMails, serializeMailStates } from './mail_flags'; const DEFAULT_NAME_REGEX = /[^a-z0-9]/g; const DEFAULT_RANDOM_SUBDOMAIN_LENGTH = 8; @@ -528,20 +528,25 @@ export const cleanup = async ( ) break; case "mails": - await c.env.DB.prepare(` - DELETE FROM raw_mails WHERE id IN ( + await deleteRawMails( + c.env.DB, + c.env, + `id IN ( SELECT id FROM raw_mails WHERE created_at < datetime('now', ?) ORDER BY created_at, id - LIMIT ? - )` - ).bind(`-${cleanDays} day`, cleanupBatchSize).run(); + LIMIT ?)`, + [`-${cleanDays} day`, cleanupBatchSize], + ); break; case "mails_unknow": - await c.env.DB.prepare(` - DELETE FROM raw_mails WHERE address NOT IN - (select name from address) AND created_at < datetime('now', '-${cleanDays} day')` - ).run(); + await deleteRawMails( + c.env.DB, + c.env, + `address NOT IN (select name from address)` + + ` AND created_at < datetime('now', '-${cleanDays} day')`, + [], + ); break; case "sendbox": await c.env.DB.prepare(` @@ -570,10 +575,12 @@ const batchDeleteAddressWithData = async ( c: Context, addressQueryCondition: string, ): Promise => { - await c.env.DB.prepare( - `DELETE FROM raw_mails WHERE address IN ( ` + - `SELECT name FROM address WHERE ${addressQueryCondition})` - ).run(); + await deleteRawMails( + c.env.DB, + c.env, + `address IN (SELECT name FROM address WHERE ${addressQueryCondition})`, + [], + ); await c.env.DB.prepare( `DELETE FROM sendbox WHERE address IN ( ` + `SELECT name FROM address WHERE ${addressQueryCondition})` @@ -627,9 +634,12 @@ export const deleteAddressWithData = async ( // unbind telegram await unbindTelegramByAddress(c, address); // delete address and related data - const { success: mailSuccess } = await c.env.DB.prepare( - `DELETE FROM raw_mails WHERE address = ? ` - ).bind(address).run(); + const { success: mailSuccess } = await deleteRawMails( + c.env.DB, + c.env, + `address = ?`, + [address], + ); const { success: sendAccess } = await c.env.DB.prepare( `DELETE FROM address_sender WHERE address = ? ` ).bind(address).run(); @@ -705,7 +715,7 @@ export const hideObjectFields = >( */ export const handleMailListQuery = async ( c: Context, - query: string, countQuery: string, params: string[], + query: string, countQuery: string, params: (string | number)[], limit: string | number | undefined | null, offset: string | number | undefined | null, orderBy?: string @@ -722,10 +732,7 @@ export const handleMailListQuery = async ( ...params, limit, offset ).all(); const resolvedResults = await resolveRawEmailList(results); - const serializedResults = resolvedResults.map(row => serializeMailState( - row, - c.env, - )); + const serializedResults = await serializeMailStates(c.env.DB, resolvedResults, c.env); const count = offset == 0 ? await c.env.DB.prepare( countQuery ).bind(...params).first("count") : 0; diff --git a/worker/src/mail_flags.ts b/worker/src/mail_flags.ts index 79e91dd..3278eeb 100644 --- a/worker/src/mail_flags.ts +++ b/worker/src/mail_flags.ts @@ -1,13 +1,11 @@ -import { getBooleanValue } from './utils'; - -export const MAIL_FLAGS = { - UNREAD: 1 << 0, - ANSWERED: 1 << 1, - FLAGGED: 1 << 2, - DELETED: 1 << 3, - DRAFT: 1 << 4, - JUNK: 1 << 5, -} as const; +export enum MailFlag { + UNREAD = 0, + ANSWERED = 1, + FLAGGED = 2, + DELETED = 3, + DRAFT = 4, + JUNK = 5, +} export const CUSTOM_MAIL_FLAG_OFFSET = 10; export const CUSTOM_MAIL_FLAG_COUNT = 10; @@ -26,9 +24,14 @@ export type MailStateOption = { default?: boolean; }; +type MailStateMutation = { + add?: number; + remove?: number[]; +}; + export type MailStateDefinition = MailStateOption & { - filter?: { mask: number; set: boolean }; - mutation?: { add: number; remove: number }; + filter?: { flag: number; present: boolean }; + mutation?: MailStateMutation; }; const SYSTEM_MAIL_STATES: MailStateDefinition[] = [ @@ -37,15 +40,15 @@ const SYSTEM_MAIL_STATES: MailStateDefinition[] = [ value: MailState.UNREAD, label_key: 'unread', unread: true, - filter: { mask: MAIL_FLAGS.UNREAD, set: true }, - mutation: { add: MAIL_FLAGS.UNREAD, remove: 0 }, + filter: { flag: MailFlag.UNREAD, present: true }, + mutation: { add: MailFlag.UNREAD }, }, { value: MailState.READ, label_key: 'read', unread: false, - filter: { mask: MAIL_FLAGS.UNREAD, set: false }, - mutation: { add: 0, remove: MAIL_FLAGS.UNREAD }, + filter: { flag: MailFlag.UNREAD, present: false }, + mutation: { remove: [MailFlag.UNREAD] }, }, ]; @@ -68,9 +71,9 @@ const getMailStateDefinition = ( export const getCustomMailFlag = (slot: number): number => { if (!Number.isInteger(slot) || slot < 0 || slot >= CUSTOM_MAIL_FLAG_COUNT) { - throw new Error("Invalid custom mail flag slot"); + throw new Error('Invalid custom mail flag slot'); } - return 1 << (CUSTOM_MAIL_FLAG_OFFSET + slot); + return CUSTOM_MAIL_FLAG_OFFSET + slot; }; export type CustomMailStateConfig = { @@ -78,7 +81,10 @@ export type CustomMailStateConfig = { name: string; }; -const CUSTOM_MAIL_FLAGS_MASK = ((1 << CUSTOM_MAIL_FLAG_COUNT) - 1) << CUSTOM_MAIL_FLAG_OFFSET; +const CUSTOM_MAIL_FLAGS = Array.from( + { length: CUSTOM_MAIL_FLAG_COUNT }, + (_, slot) => getCustomMailFlag(slot), +); export const createCustomMailStateDefinitions = ( configs: CustomMailStateConfig[], @@ -88,23 +94,58 @@ export const createCustomMailStateDefinitions = ( return { value: `custom:${config.slot}`, label: config.name, - filter: { mask: flag, set: true }, - mutation: { add: flag, remove: CUSTOM_MAIL_FLAGS_MASK & ~flag }, + filter: { flag, present: true }, + mutation: { add: flag, remove: CUSTOM_MAIL_FLAGS.filter(value => value !== flag) }, }; }); }; -export const serializeMailState = >( +const withoutLegacyFlags = >(row: T): T => { + const result = { ...row }; + delete result.flags; + return result; +}; + +const isMailFlagsEnabled = (env: Bindings): boolean => { + return env.ENABLE_MAIL_FLAGS === true || env.ENABLE_MAIL_FLAGS === 'true'; +}; + +export const serializeMailStates = async >( + db: D1Database, + rows: T[], + env: Bindings, +): Promise => { + const results = rows.map(withoutLegacyFlags); + if (!isMailFlagsEnabled(env) || results.length === 0) return results; + + const needsUnread = (row: T) => ![true, false, 0, 1].includes(row.unread as boolean | number); + const ids = [...new Set(results.filter(needsUnread).map(row => Number(row.id)))] + .filter(id => Number.isInteger(id) && id > 0); + if (ids.length === 0) { + return results.map(row => ({ ...row, unread: Boolean(row.unread) })); + } + + const placeholders = ids.map(() => '?').join(','); + const { results: flagRows } = await db.prepare( + `SELECT mf.mail_id FROM mail_flags mf` + + ` JOIN raw_mails rm ON rm.id = mf.mail_id` + + ` JOIN address a ON a.id = mf.address_id AND a.name = rm.address` + + ` WHERE mf.flag = ? AND mf.mail_id IN (${placeholders})` + ).bind(MailFlag.UNREAD, ...ids).all<{ mail_id: number }>(); + const unreadIds = new Set(flagRows.map(row => Number(row.mail_id))); + + return results.map(row => ({ + ...row, + unread: needsUnread(row) ? unreadIds.has(Number(row.id)) : Boolean(row.unread), + })); +}; + +export const serializeMailState = async >( + db: D1Database, row: T, env: Bindings, -): T => { - const result = { ...row }; - const flags = Number(result.flags ?? 0); - delete result.flags; - if (!getBooleanValue(env.ENABLE_MAIL_FLAGS)) { - return result; - } - result.unread = (flags & MAIL_FLAGS.UNREAD) !== 0; +): Promise => { + const [result] = await serializeMailStates(db, [row], env); return result; }; @@ -112,8 +153,8 @@ const resolveInitialMailFlags = async ( _env: Bindings, _address: string, _parsedEmailContext: ParsedEmailContext, -): Promise => { - return MAIL_FLAGS.UNREAD; +): Promise => { + return [MailFlag.UNREAD]; }; export const updateInitialMailFlags = async ( @@ -125,43 +166,62 @@ export const updateInitialMailFlags = async ( parsedEmailContext: ParsedEmailContext, ) => { if (!enabled || !Number.isInteger(mailId) || mailId <= 0) return; + const flags = await resolveInitialMailFlags(env, address, parsedEmailContext); - await db.prepare(`UPDATE raw_mails SET flags = ? WHERE id = ?`).bind(flags, mailId).run(); + if (flags.length === 0) return; + + const selects = flags.map(() => 'SELECT ?, id, ? FROM address WHERE name = ?').join(' UNION ALL '); + await db.prepare( + `INSERT OR IGNORE INTO mail_flags (mail_id, address_id, flag) ${selects}` + ).bind(...flags.flatMap(flag => [mailId, flag, address])).run(); }; -export type MailStateUpdate = { - ids: number[]; - state: string; - add: number; - remove: number; -}; - -export type MailReadStatusQuery = { - clause: string; - params: string[]; +export type MailStateQuery = { + join: string; + clause?: string; + orderBy?: string; + unread?: boolean; + params: number[]; }; export const getMailStateQuery = ( value: string | undefined, - column: 'flags' | 'rm.flags', + mailAlias: string, + addressIdColumn: string, customStates: MailStateDefinition[] = [], -): MailReadStatusQuery | undefined | null => { +): MailStateQuery | undefined | null => { if (value === undefined) return undefined; + const definition = getMailStateDefinition(value, customStates); if (!definition) return null; if (!definition.filter) return undefined; - const operator = definition.filter.set ? '!=' : '='; + + const { flag, present } = definition.filter; + const joinType = present ? 'JOIN' : 'LEFT JOIN'; return { - clause: `(COALESCE(${column}, 0) & ?) ${operator} 0`, - params: [String(definition.filter.mask)], + join: ` ${joinType} mail_flags mail_state_flags` + + ` ON mail_state_flags.mail_id = ${mailAlias}.id` + + ` AND mail_state_flags.address_id = ${addressIdColumn}` + + ` AND mail_state_flags.flag = ?`, + clause: present ? undefined : 'mail_state_flags.mail_id IS NULL', + orderBy: present ? 'mail_state_flags.mail_id desc' : undefined, + unread: definition.unread, + params: [flag], }; }; +type MailStateUpdate = { + ids: number[]; + mutation: MailStateMutation; + unread?: boolean; +}; + const parseMailStateUpdate = ( value: unknown, customStates: MailStateDefinition[] = [], ): MailStateUpdate | null => { if (!value || typeof value !== 'object') return null; + const body = value as Record; if (!Array.isArray(body.ids) || body.ids.length === 0 || body.ids.length > 100) return null; if (body.ids.some(id => typeof id !== 'number')) return null; @@ -171,24 +231,7 @@ const parseMailStateUpdate = ( const definition = getMailStateDefinition(body.state, customStates); if (!definition?.mutation) return null; - return { - ids, - state: definition.value, - add: definition.mutation.add, - remove: definition.mutation.remove, - }; -}; - -const getMailStateUpdateExpression = ( - update: MailStateUpdate, - column = 'flags', -): { expression: string; params: number[]; condition?: string; conditionParams?: number[] } => { - return { - expression: `((COALESCE(${column}, 0) | ?) & ~?)`, - params: [update.add, update.remove], - condition: `((COALESCE(${column}, 0) & ?) != ? OR (COALESCE(${column}, 0) & ?) != 0)`, - conditionParams: [update.add, update.add, update.remove], - }; + return { ids, mutation: definition.mutation, unread: definition.unread }; }; type MailScope = { @@ -206,27 +249,87 @@ export const applyMailStateUpdate = async ( const update = parseMailStateUpdate(value, customStates); if (!update) return null; - const placeholders = update.ids.map(() => '?').join(','); - const statusUpdate = getMailStateUpdateExpression(update); - const condition = statusUpdate.condition ? ` AND ${statusUpdate.condition}` : ''; - const result = await db.prepare( - `UPDATE raw_mails SET flags = ${statusUpdate.expression}` - + ` WHERE id IN (${placeholders}) AND (${scope.clause})${condition}` - ).bind( - ...statusUpdate.params, - ...update.ids, - ...scope.params, - ...(statusUpdate.conditionParams ?? []), - ).run(); - if (!result.success) return { success: false, changes: 0, results: [] }; + const idPlaceholders = update.ids.map(() => '?').join(','); + const targetWhere = `rm.id IN (${idPlaceholders}) AND (${scope.clause})`; + const statements: D1PreparedStatement[] = []; + if (update.mutation.remove?.length) { + const flagPlaceholders = update.mutation.remove.map(() => '?').join(','); + statements.push(db.prepare( + `DELETE FROM mail_flags WHERE flag IN (${flagPlaceholders})` + + ` AND mail_id IN (` + + `SELECT rm.id FROM raw_mails rm JOIN address a ON a.name = rm.address` + + ` WHERE ${targetWhere})` + ).bind(...update.mutation.remove, ...update.ids, ...scope.params)); + } + + if (update.mutation.add !== undefined) { + statements.push(db.prepare( + `INSERT OR IGNORE INTO mail_flags (mail_id, address_id, flag)` + + ` SELECT rm.id, a.id, ?` + + ` FROM raw_mails rm JOIN address a ON a.name = rm.address` + + ` WHERE ${targetWhere}` + ).bind(update.mutation.add, ...update.ids, ...scope.params)); + } + + const mutationResults = await db.batch(statements); + if (mutationResults.some(result => !result.success)) { + return { success: false, changes: 0, results: [] }; + } + + const unreadSelect = update.unread === undefined ? '' : `, ${update.unread ? 1 : 0} AS unread`; const { results } = await db.prepare( - `SELECT id, flags FROM raw_mails` - + ` WHERE id IN (${placeholders}) AND (${scope.clause})` + `SELECT rm.id${unreadSelect} FROM raw_mails rm JOIN address a ON a.name = rm.address` + + ` WHERE ${targetWhere}` ).bind(...update.ids, ...scope.params).all(); + return { success: true, - changes: result.meta.changes ?? 0, - results: results.map(row => serializeMailState(row, env)), + changes: mutationResults.reduce((total, result) => total + (result.meta.changes ?? 0), 0), + results: await serializeMailStates(db, results, env), }; }; + +export const prepareRawMailDeleteStatements = ( + db: D1Database, + env: Bindings, + whereClause: string, + params: (string | number)[], +): D1PreparedStatement[] => { + const deleteMail = db.prepare( + `DELETE FROM raw_mails WHERE ${whereClause}` + ).bind(...params); + if (!isMailFlagsEnabled(env)) return [deleteMail]; + + const deleteFlags = db.prepare( + `DELETE FROM mail_flags WHERE mail_id IN (` + + `SELECT id FROM raw_mails WHERE ${whereClause})` + ).bind(...params); + return [deleteFlags, deleteMail]; +}; + +export const deleteRawMails = async ( + db: D1Database, + env: Bindings, + whereClause: string, + params: (string | number)[], +): Promise => { + const results = await db.batch(prepareRawMailDeleteStatements(db, env, whereClause, params)); + return results[results.length - 1]; +}; + +export const cleanupOrphanMailFlags = async ( + db: D1Database, + env: Bindings, + limit = 1000, +): Promise => { + if (!isMailFlagsEnabled(env)) return 0; + + const result = await db.prepare( + `DELETE FROM mail_flags WHERE (mail_id, flag) IN (` + + `SELECT mf.mail_id, mf.flag FROM mail_flags mf` + + ` LEFT JOIN raw_mails rm ON rm.id = mf.mail_id` + + ` WHERE rm.id IS NULL LIMIT ?)` + ).bind(limit).run(); + return result.meta.changes ?? 0; +}; diff --git a/worker/src/mails_api/mails_crud.ts b/worker/src/mails_api/mails_crud.ts index 6979bcb..53bd637 100644 --- a/worker/src/mails_api/mails_crud.ts +++ b/worker/src/mails_api/mails_crud.ts @@ -10,6 +10,7 @@ import { getMailStateOptions, applyMailStateUpdate, serializeMailState, + deleteRawMails, } from '../mail_flags'; const listMails = async (c: Context) => { @@ -19,23 +20,33 @@ const listMails = async (c: Context) => { } const { limit, offset, mail_state } = c.req.query(); if (Number.parseInt(offset) <= 0) updateAddressUpdatedAt(c, address); - const stateQuery = getMailStateQuery(mail_state, 'flags'); + const stateQuery = getMailStateQuery(mail_state, 'rm', 'a.id'); if (stateQuery === null) return c.json({ error: "Invalid mail state filter" }, 400); if (stateQuery && !getBooleanValue(c.env.ENABLE_MAIL_FLAGS)) { return c.json({ error: "Mail states are disabled" }, 403); } - const filters = [`address = ?`]; - const params = [address]; - if (stateQuery) { - filters.push(stateQuery.clause); - params.push(...stateQuery.params); + if (!stateQuery) { + return await handleMailListQuery(c, + `SELECT * FROM raw_mails WHERE address = ?`, + `SELECT count(*) as count FROM raw_mails WHERE address = ?`, + [address], limit, offset + ); } - const whereClause = filters.join(' AND '); + + const filters = [`rm.address = ?`]; + if (stateQuery.clause) filters.push(stateQuery.clause); + const fromQuery = ` FROM raw_mails rm` + + ` JOIN address a ON a.name = rm.address` + + stateQuery.join + + ` WHERE ${filters.join(' AND ')}`; + const unreadSelect = stateQuery.unread === undefined + ? '' + : `, ${stateQuery.unread ? 1 : 0} AS unread`; return await handleMailListQuery(c, - `SELECT * FROM raw_mails WHERE ${whereClause}`, - `SELECT count(*) as count FROM raw_mails WHERE ${whereClause}`, - params, limit, offset + `SELECT rm.*${unreadSelect}${fromQuery}`, + `SELECT count(*) as count${fromQuery}`, + [...stateQuery.params, address], limit, offset, stateQuery.orderBy ?? 'rm.id desc' ); }; @@ -46,7 +57,8 @@ const getMail = async (c: Context) => { `SELECT * FROM raw_mails where id = ? and address = ?` ).bind(mail_id, address).first(); if (!result) return c.json(null); - return c.json(serializeMailState( + return c.json(await serializeMailState( + c.env.DB, await resolveRawEmailRow(result), c.env, )); @@ -60,9 +72,12 @@ const deleteMail = async (c: Context) => { const { address } = c.get("jwtPayload") const { id } = c.req.param(); // TODO: add toLowerCase() to handle old data - const { success } = await c.env.DB.prepare( - `DELETE FROM raw_mails WHERE address = ? and id = ? ` - ).bind(address.toLowerCase(), id).run(); + const { success } = await deleteRawMails( + c.env.DB, + c.env, + `address = ? and id = ?`, + [address.toLowerCase(), id], + ); return c.json({ success }); }; @@ -74,7 +89,7 @@ const updateMailState = async (c: Context) => { const result = await applyMailStateUpdate( c.env.DB, c.env, - { clause: 'address = ?', params: [address] }, + { clause: 'rm.address = ?', params: [address] }, await c.req.json().catch(() => null), ); if (!result) return c.json({ error: "Invalid mail state request" }, 400); @@ -138,9 +153,12 @@ const clearInbox = async (c: Context) => { return c.text(msgs.UserDeleteEmailDisabledMsg, 403) } const { address } = c.get("jwtPayload") - const { success } = await c.env.DB.prepare( - `DELETE FROM raw_mails WHERE address = ?` - ).bind(address).run(); + const { success } = await deleteRawMails( + c.env.DB, + c.env, + `address = ?`, + [address], + ); if (!success) { return c.text(msgs.FailedClearInboxMsg, 500) } diff --git a/worker/src/mails_api/parsed_mail_api.ts b/worker/src/mails_api/parsed_mail_api.ts index df91738..7f43b20 100644 --- a/worker/src/mails_api/parsed_mail_api.ts +++ b/worker/src/mails_api/parsed_mail_api.ts @@ -46,7 +46,8 @@ const getParsedMail = async (c: Context) => { `SELECT * FROM raw_mails where id = ? and address = ?` ).bind(mail_id, address).first(); if (!row) return c.json(null); - const resolved = serializeMailState( + const resolved = await serializeMailState( + c.env.DB, await resolveRawEmailRow(row), c.env, ); diff --git a/worker/src/models/index.ts b/worker/src/models/index.ts index e951db6..80bc04e 100644 --- a/worker/src/models/index.ts +++ b/worker/src/models/index.ts @@ -213,7 +213,6 @@ export type RawMailRow = { raw?: string; raw_blob?: unknown; metadata?: string; - flags?: number | null; unread?: boolean; created_at?: string; } diff --git a/worker/src/telegram_api/miniapp.ts b/worker/src/telegram_api/miniapp.ts index e9d91bc..5816488 100644 --- a/worker/src/telegram_api/miniapp.ts +++ b/worker/src/telegram_api/miniapp.ts @@ -146,7 +146,7 @@ async function getMail(c: Context): Promise { if (!result) { return c.text("Mail not found", 404); } - return c.json(serializeMailState(await resolveRawEmailRow(result), c.env)); + return c.json(await serializeMailState(c.env.DB, await resolveRawEmailRow(result), c.env)); } const userId = await checkTelegramAuth(c, initData); const jwtList = await c.env.KV.get(`${CONSTANTS.TG_KV_PREFIX}:${userId}`, 'json') || []; @@ -169,7 +169,7 @@ async function getMail(c: Context): Promise { return c.text(msgs.TgNoPermissionViewMailMsg, 403); } } - return c.json(serializeMailState(await resolveRawEmailRow(result), c.env)); + return c.json(await serializeMailState(c.env.DB, await resolveRawEmailRow(result), c.env)); } catch (e) { return c.text((e as Error).message, 400); diff --git a/worker/src/user_api/user_mail_api.ts b/worker/src/user_api/user_mail_api.ts index efab1e2..b501268 100644 --- a/worker/src/user_api/user_mail_api.ts +++ b/worker/src/user_api/user_mail_api.ts @@ -6,6 +6,7 @@ import { getMailStateQuery, getMailStateOptions, applyMailStateUpdate, + deleteRawMails, } from "../mail_flags"; export default { @@ -24,23 +25,24 @@ export default { filterQuerys.push(`rm.address = ?`); filterParams.push(address); } - const stateQuery = getMailStateQuery(mail_state, 'rm.flags'); + const stateQuery = getMailStateQuery(mail_state, 'rm', 'a.id'); if (stateQuery === null) return c.json({ error: "Invalid mail state filter" }, 400); if (stateQuery && !getBooleanValue(c.env.ENABLE_MAIL_FLAGS)) { return c.json({ error: "Mail states are disabled" }, 403); } - if (stateQuery) { - filterQuerys.push(stateQuery.clause); - filterParams.push(...stateQuery.params); - } + if (stateQuery?.clause) filterQuerys.push(stateQuery.clause); const fromQuery = ` FROM users_address ua` + ` JOIN address a ON a.id = ua.address_id` + ` JOIN raw_mails rm ON rm.address = a.name` + + (stateQuery?.join ?? '') + ` WHERE ${filterQuerys.join(" AND ")}`; + const unreadSelect = stateQuery?.unread === undefined + ? '' + : `, ${stateQuery.unread ? 1 : 0} AS unread`; return await handleMailListQuery(c, - `SELECT rm.*${fromQuery}`, + `SELECT rm.*${unreadSelect}${fromQuery}`, `SELECT count(*) as count${fromQuery}`, - filterParams, limit, offset, 'rm.id desc' + [...(stateQuery?.params ?? []), ...filterParams], limit, offset, 'rm.id desc' ); }, deleteMail: async (c: Context) => { @@ -50,14 +52,17 @@ export default { } const { id } = c.req.param(); const { user_id } = c.get("userPayload"); - const { success } = await c.env.DB.prepare( - `DELETE FROM raw_mails WHERE id = ?` + const { success } = await deleteRawMails( + c.env.DB, + c.env, + `id = ?` + ` AND EXISTS (` + `SELECT 1 FROM users_address ua` + ` JOIN address a ON a.id = ua.address_id` + ` WHERE ua.user_id = ? AND a.name = raw_mails.address` - + `)` - ).bind(id, user_id).run(); + + `)`, + [id, user_id], + ); return c.json({ success: success }) @@ -71,11 +76,9 @@ export default { c.env.DB, c.env, { - clause: `EXISTS (` - + `SELECT 1 FROM users_address ua` - + ` JOIN address a ON a.id = ua.address_id` - + ` WHERE ua.user_id = ? AND a.name = raw_mails.address` - + `)`, + clause: `a.id IN (` + + `SELECT address_id FROM users_address WHERE user_id = ?` + + `)`, params: [user_id], }, await c.req.json().catch(() => null),