diff --git a/CHANGELOG.md b/CHANGELOG.md index c6b69169..9bdb25b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ - feat: 增加 worker 配置 `DISABLE_ANONYMOUS_USER_CREATE_EMAIL` 禁用匿名用户创建邮箱地址，只允许登录用户创建邮箱地址 - feat: 增加 worker 配置 `ENABLE_ANOTHER_WORKER` 及 `ANOTHER_WORKER_LIST` ，用于调用其他 worker 的 rpc 接口 (#547) - feat: |UI| 自动刷新配置保存到浏览器，可配置刷新间隔 +- feat: 垃圾邮件检测增加存在是才检查的列表 `JUNK_MAIL_CHECK_LIST` 配置 +- feat: | Worker | 增加 `ParsedEmailContext` 类用于缓存解析后的邮件内容，减少解析次数 ## v0.8.3 diff --git a/vitepress-docs/docs/en/cli.md b/vitepress-docs/docs/en/cli.md index 1e173f11..b32413a7 100644 --- a/vitepress-docs/docs/en/cli.md +++ b/vitepress-docs/docs/en/cli.md @@ -134,6 +134,8 @@ ENABLE_AUTO_REPLY = false # FRONTEND_URL = "https://xxxx.xxx" # Enable check junk mail # ENABLE_CHECK_JUNK_MAIL = false +# junk mail check list, if status exists and status is not pass, will be marked as junk mail +# JUNK_MAIL_CHECK_LIST = = ["spf", "dkim", "dmarc"] # junk mail force check pass list, if no status or status is not pass, will be marked as junk mail # JUNK_MAIL_FORCE_PASS_LIST = ["spf", "dkim", "dmarc"] diff --git a/vitepress-docs/docs/zh/guide/cli/worker.md b/vitepress-docs/docs/zh/guide/cli/worker.md index e99e44ce..b2bfc022 100644 --- a/vitepress-docs/docs/zh/guide/cli/worker.md +++ b/vitepress-docs/docs/zh/guide/cli/worker.md @@ -106,7 +106,9 @@ ENABLE_AUTO_REPLY = false # FRONTEND_URL = "https://xxxx.xxx" # 是否启用垃圾邮件检查，默认任何一项存在配置且不通过则被判定为垃圾邮件 # ENABLE_CHECK_JUNK_MAIL = false -# 垃圾邮件检查配置, 任何一项不存在或者不通过则被判定为垃圾邮件 +# 垃圾邮件检查配置, 任何一项 存在 且 不通过 则被判定为垃圾邮件 +# JUNK_MAIL_CHECK_LIST = = ["spf", "dkim", "dmarc"] +# 垃圾邮件检查配置, 任何一项 不存在 或者 不通过 则被判定为垃圾邮件 # JUNK_MAIL_FORCE_PASS_LIST = ["spf", "dkim", "dmarc"] # 是否开启其他 worker 处理邮件 # ENABLE_ANOTHER_WORKER = false diff --git a/worker/src/admin_api/mail_webhook_settings.ts b/worker/src/admin_api/mail_webhook_settings.ts index 8391dced..a3c49eea 100644 --- a/worker/src/admin_api/mail_webhook_settings.ts +++ b/worker/src/admin_api/mail_webhook_settings.ts @@ -1,5 +1,5 @@ import { Context } from "hono"; -import { HonoCustomType } from "../types"; +import { HonoCustomType, ParsedEmailContext } from "../types"; import { CONSTANTS } from "../constants"; import { WebhookSettings } from "../models"; import { commonParseMail, sendWebhook } from "../common"; @@ -25,8 +25,8 @@ async function testWebhookSettings(c: Context): Promise() || {}; - - const parsedEmail = await commonParseMail(raw); + const parsedEmailContext: ParsedEmailContext = { rawEmail: raw || "" }; + const parsedEmail = await commonParseMail(parsedEmailContext); const res = await sendWebhook(settings, { id: mailId || "0", url: c.env.FRONTEND_URL ? `${c.env.FRONTEND_URL}?mail_id=${mailId}` : "", diff --git a/worker/src/admin_api/worker_config.ts b/worker/src/admin_api/worker_config.ts index ecec5462..14b9af5b 100644 --- a/worker/src/admin_api/worker_config.ts +++ b/worker/src/admin_api/worker_config.ts @@ -43,6 +43,7 @@ export default { "DISABLE_SHOW_GITHUB": !getBooleanValue(c.env.DISABLE_SHOW_GITHUB), "DISABLE_ADMIN_PASSWORD_CHECK": getBooleanValue(c.env.DISABLE_ADMIN_PASSWORD_CHECK), "ENABLE_CHECK_JUNK_MAIL": getBooleanValue(c.env.ENABLE_CHECK_JUNK_MAIL), + "JUNK_MAIL_CHECK_LIST": getStringArray(c.env.JUNK_MAIL_CHECK_LIST), "JUNK_MAIL_FORCE_PASS_LIST": getStringArray(c.env.JUNK_MAIL_FORCE_PASS_LIST), "ENABLE_ANOTHER_WORKER": getBooleanValue(c.env.ENABLE_ANOTHER_WORKER), diff --git a/worker/src/common.ts b/worker/src/common.ts index ace83862..8b63e8fc 100644 --- a/worker/src/common.ts +++ b/worker/src/common.ts @@ -2,7 +2,7 @@ import { Context } from 'hono'; import { Jwt } from 'hono/utils/jwt' import { getBooleanValue, getDomains, getStringValue, getIntValue, getUserRoles, getDefaultDomains, getJsonSetting, getAnotherWorkerList } from './utils'; -import { HonoCustomType, UserRole, AnotherWorker, RPCEmailMessage } from './types'; +import { HonoCustomType, UserRole, AnotherWorker, RPCEmailMessage, ParsedEmailContext } from './types'; import { unbindTelegramByAddress } from './telegram_api/common'; import { CONSTANTS } from './constants'; import { AdminWebhookSettings, WebhookMail, WebhookSettings } from './models'; @@ -256,16 +256,22 @@ export const handleListQuery = async ( } -export const commonParseMail = async (raw_mail: string | undefined | null): Promise<{ +export const commonParseMail = async (parsedEmailContext: ParsedEmailContext): Promise<{ sender: string, subject: string, text: string, html: string, headers?: Record[] } | undefined> => { - if (!raw_mail) { + // check parsed email context is valid + if (!parsedEmailContext || !parsedEmailContext.rawEmail) { return undefined; } + // return parsed email if already parsed + if (parsedEmailContext.parsedEmail) { + return parsedEmailContext.parsedEmail; + } + const raw_mail = parsedEmailContext.rawEmail; // TODO: WASM parse email // try { // const { parse_message_wrapper } = await import('mail-parser-wasm-worker'); @@ -275,7 +281,9 @@ export const commonParseMail = async (raw_mail: string | undefined | null): Prom // sender: parsedEmail.sender || "", // subject: parsedEmail.subject || "", // text: parsedEmail.text || "", - // headers: parsedEmail.headers || [], + // headers: parsedEmail.headers?.map( + // (header) => ({ key: header.key, value: header.value }) + // ) || [], // html: parsedEmail.body_html || "", // }; // } catch (e) { @@ -358,9 +366,9 @@ export async function sendWebhook(settings: WebhookSettings, formatMap: WebhookM export async function triggerWebhook( c: Context, address: string, - raw_mail: string, + parsedEmailContext: ParsedEmailContext, message_id: string | null -): Promise { +): Promise { if (!c.env.KV || !getBooleanValue(c.env.ENABLE_WEBHOOK)) { return } @@ -391,14 +399,14 @@ export async function triggerWebhook( `SELECT id FROM raw_mails where address = ? and message_id = ?` ).bind(address, message_id).first("id"); - const parsedEmail = await commonParseMail(raw_mail); + const parsedEmail = await commonParseMail(parsedEmailContext); const webhookMail = { id: mailId || "", url: c.env.FRONTEND_URL ? `${c.env.FRONTEND_URL}?mail_id=${mailId}` : "", from: parsedEmail?.sender || "", to: address, subject: parsedEmail?.subject || "", - raw: raw_mail, + raw: parsedEmailContext.rawEmail || "", parsedText: parsedEmail?.text || "", parsedHtml: parsedEmail?.html || "" } @@ -408,7 +416,6 @@ export async function triggerWebhook( console.error(res.message); } } - return webhookMail.parsedText } export async function triggerAnotherWorker( diff --git a/worker/src/email/check_junk.ts b/worker/src/email/check_junk.ts index 49a95389..59919c52 100644 --- a/worker/src/email/check_junk.ts +++ b/worker/src/email/check_junk.ts @@ -1,19 +1,22 @@ -import { Bindings } from "../types"; +import { Bindings, ParsedEmailContext } from "../types"; import { getBooleanValue, getStringArray } from "../utils"; import { commonParseMail } from "../common"; export const check_if_junk_mail = async ( env: Bindings, address: string, - raw_mail: string, message_id: string | null + parsedEmailContext: ParsedEmailContext, + message_id: string | null ): Promise => { if (!getBooleanValue(env.ENABLE_CHECK_JUNK_MAIL)) { return false; } - const parsedEmail = await commonParseMail(raw_mail); + const parsedEmail = await commonParseMail(parsedEmailContext); if (!parsedEmail?.headers) return false; + const checkListWhenExist = getStringArray(env.JUNK_MAIL_CHECK_LIST); const forcePassList = getStringArray(env.JUNK_MAIL_FORCE_PASS_LIST); const passedList: string[] = []; + const existList: string[] = []; const headers = parsedEmail.headers; for (const header of headers) { @@ -22,28 +25,35 @@ export const check_if_junk_mail = async ( // check spf if (header["key"].toLowerCase() == "received-spf") { - if (!header["value"].toLowerCase().includes("pass")) { - return true; + existList.push("spf"); + if (header["value"].toLowerCase().includes("pass")) { + passedList.push("spf"); } - passedList.push("spf"); } // check dkim and dmarc if (header["key"].toLowerCase() == "authentication-results") { if (header["value"].toLowerCase().includes("dkim=")) { - if (!header["value"].toLowerCase().includes("dkim=pass")) { - return true; + existList.push("dkim"); + if (header["value"].toLowerCase().includes("dkim=pass")) { + passedList.push("dkim"); } - passedList.push("dkim"); } if (header["value"].toLowerCase().includes("dmarc=")) { - if (!header["value"].toLowerCase().includes("dmarc=pass")) { - return true; + existList.push("dmarc"); + if (header["value"].toLowerCase().includes("dmarc=pass")) { + passedList.push("dmarc"); } - passedList.push("dmarc"); } } } + // check if all checkListWhenExist item passed when exist + if (checkListWhenExist?.some( + (checkName) => existList.includes(checkName.toLowerCase()) + && !passedList.includes(checkName.toLowerCase()) + )) { + return true; + } if (forcePassList?.length == 0) return false; diff --git a/worker/src/email/index.ts b/worker/src/email/index.ts index ff2a344c..bd04636d 100644 --- a/worker/src/email/index.ts +++ b/worker/src/email/index.ts @@ -2,10 +2,10 @@ import { Context } from "hono"; import { getEnvStringList } from "../utils"; import { sendMailToTelegram } from "../telegram_api"; -import { Bindings, HonoCustomType, RPCEmailMessage } from "../types"; +import { Bindings, HonoCustomType, RPCEmailMessage, ParsedEmailContext } from "../types"; import { auto_reply } from "./auto_reply"; import { isBlocked } from "./black_list"; -import { triggerWebhook, triggerAnotherWorker, commonParseMail} from "../common"; +import { triggerWebhook, triggerAnotherWorker, commonParseMail } from "../common"; import { check_if_junk_mail } from "./check_junk"; @@ -16,10 +16,13 @@ async function email(message: ForwardableEmailMessage, env: Bindings, ctx: Execu return; } const rawEmail = await new Response(message.raw).text(); + const parsedEmailContext: ParsedEmailContext = { + rawEmail: rawEmail + }; // check if junk mail try { - const is_junk = await check_if_junk_mail(env, message.to, rawEmail, message.headers.get("Message-ID")); + const is_junk = await check_if_junk_mail(env, message.to, parsedEmailContext, message.headers.get("Message-ID")); if (is_junk) { message.setReject("Junk mail"); console.log(`Junk mail from ${message.from} to ${message.to}`); @@ -31,14 +34,19 @@ async function email(message: ForwardableEmailMessage, env: Bindings, ctx: Execu const message_id = message.headers.get("Message-ID"); // save email - const { success } = await env.DB.prepare( - `INSERT INTO raw_mails (source, address, raw, message_id) VALUES (?, ?, ?, ?)` - ).bind( - message.from, message.to, rawEmail, message_id - ).run(); - if (!success) { - message.setReject(`Failed save message to ${message.to}`); - console.log(`Failed save message from ${message.from} to ${message.to}`); + try { + const { success } = await env.DB.prepare( + `INSERT INTO raw_mails (source, address, raw, message_id) VALUES (?, ?, ?, ?)` + ).bind( + message.from, message.to, rawEmail, message_id + ).run(); + if (!success) { + message.setReject(`Failed save message to ${message.to}`); + console.log(`Failed save message from ${message.from} to ${message.to}`); + } + } + catch (error) { + console.log("save email error", error); } // forward email @@ -55,17 +63,16 @@ async function email(message: ForwardableEmailMessage, env: Bindings, ctx: Execu try { await sendMailToTelegram( { env: env } as Context, - message.to, rawEmail, message_id); + message.to, parsedEmailContext, message_id); } catch (error) { console.log("send mail to telegram error", error); } // send webhook - let parsedText; try { - parsedText = await triggerWebhook( + await triggerWebhook( { env: env } as Context, - message.to, rawEmail, message_id + message.to, parsedEmailContext, message_id ); } catch (error) { console.log("send webhook error", error); @@ -74,12 +81,10 @@ async function email(message: ForwardableEmailMessage, env: Bindings, ctx: Execu // trigger another worker try { const headersMap = new Map(); - if(message.headers) { - message.headers.forEach((value, key) => {headersMap.set(key, value);}); - } - if (!parsedText){ - parsedText = (await commonParseMail(rawEmail))?.text ?? "" + if (message.headers) { + message.headers.forEach((value, key) => { headersMap.set(key, value); }); } + const parsedText = (await commonParseMail(parsedEmailContext))?.text ?? "" const rpcEmail: RPCEmailMessage = { from: message.from, to: message.to, diff --git a/worker/src/mails_api/webhook_settings.ts b/worker/src/mails_api/webhook_settings.ts index 9e77fc7a..bb205e8c 100644 --- a/worker/src/mails_api/webhook_settings.ts +++ b/worker/src/mails_api/webhook_settings.ts @@ -1,5 +1,5 @@ import { Context } from "hono"; -import { HonoCustomType } from "../types"; +import { HonoCustomType, ParsedEmailContext } from "../types"; import { CONSTANTS } from "../constants"; import { AdminWebhookSettings, WebhookSettings } from "../models"; import { getBooleanValue } from "../utils"; @@ -39,8 +39,8 @@ async function testWebhookSettings(c: Context): Promise() || {}; - - const parsedEmail = await commonParseMail(raw); + const parsedEmailContext: ParsedEmailContext = { rawEmail: raw || "" }; + const parsedEmail = await commonParseMail(parsedEmailContext); const res = await sendWebhook(settings, { id: mailId || "0", url: c.env.FRONTEND_URL ? `${c.env.FRONTEND_URL}?mail_id=${mailId}` : "", diff --git a/worker/src/telegram_api/telegram.ts b/worker/src/telegram_api/telegram.ts index 39a5dcf0..7d6d1f3e 100644 --- a/worker/src/telegram_api/telegram.ts +++ b/worker/src/telegram_api/telegram.ts @@ -5,7 +5,7 @@ import { callbackQuery } from "telegraf/filters"; import { CONSTANTS } from "../constants"; import { getDomains, getJsonObjectValue, getStringValue } from '../utils'; -import { HonoCustomType } from "../types"; +import { HonoCustomType, ParsedEmailContext } from "../types"; import { TelegramSettings } from "./settings"; import { bindTelegramAddress, deleteTelegramAddress, jwtListToAddressData, tgUserNewAddress, unbindTelegramAddress, unbindTelegramByAddress } from "./common"; import { commonParseMail } from "../common"; @@ -295,14 +295,14 @@ export async function initTelegramBotCommands(bot: Telegraf) { } const parseMail = async ( - raw_mail: string | undefined | null, + parsedEmailContext: ParsedEmailContext, address: string, created_at: string | undefined | null ) => { - if (!raw_mail) { + if (!parsedEmailContext.rawEmail) { return {}; } try { - const parsedEmail = await commonParseMail(raw_mail); + const parsedEmail = await commonParseMail(parsedEmailContext); let parsedText = parsedEmail?.text || ""; if (parsedText.length && parsedText.length > 1000) { parsedText = parsedEmail?.text.substring(0, 1000) + "\n\n...\n消息过长请到miniapp查看"; @@ -326,13 +326,14 @@ const parseMail = async ( export async function sendMailToTelegram( c: Context, address: string, - raw_mail: string, message_id: string | null + parsedEmailContext: ParsedEmailContext, + message_id: string | null ) { if (!c.env.TELEGRAM_BOT_TOKEN || !c.env.KV) { return; } const userId = await c.env.KV.get(`${CONSTANTS.TG_KV_PREFIX}:${address}`); - const { mail } = await parseMail(raw_mail, address, new Date().toUTCString()); + const { mail } = await parseMail(parsedEmailContext, address, new Date().toUTCString()); if (!mail) { return; } diff --git a/worker/src/types.d.ts b/worker/src/types.d.ts index 7fc7cb88..450f3201 100644 --- a/worker/src/types.d.ts +++ b/worker/src/types.d.ts @@ -44,6 +44,7 @@ export type Bindings = { FORWARD_ADDRESS_LIST: string | string[] | undefined ENABLE_CHECK_JUNK_MAIL: string | boolean | undefined + JUNK_MAIL_CHECK_LIST: string | string[] | undefined JUNK_MAIL_FORCE_PASS_LIST: string | string[] | undefined ENABLE_ANOTHER_WORKER: string | boolean | undefined @@ -108,4 +109,15 @@ type RPCEmailMessage = { to: string | undefined | null, rawEmail: string | undefined | null, headers: Map, -} \ No newline at end of file +} + +type ParsedEmailContext = { + rawEmail: string, + parsedEmail?: { + sender: string, + subject: string, + text: string, + html: string, + headers?: Record[] + } | undefined +} diff --git a/worker/wrangler.toml.template b/worker/wrangler.toml.template index 312cf3fb..464f8063 100644 --- a/worker/wrangler.toml.template +++ b/worker/wrangler.toml.template @@ -76,6 +76,8 @@ ENABLE_AUTO_REPLY = false # FRONTEND_URL = "https://xxxx.xxx" # Enable check junk mail # ENABLE_CHECK_JUNK_MAIL = false +# junk mail check list, if status exists and status is not pass, will be marked as junk mail +# JUNK_MAIL_CHECK_LIST = = ["spf", "dkim", "dmarc"] # junk mail force check pass list, if no status or status is not pass, will be marked as junk mail # JUNK_MAIL_FORCE_PASS_LIST = ["spf", "dkim", "dmarc"] # Calling other woker to process email @@ -114,4 +116,4 @@ database_id = "xxx" # binding another worker service (parse the code or link), e.g. auth-inbox # [[services]] # binding = "AUTH_INBOX" -# service = "auth-inbox" \ No newline at end of file +# service = "auth-inbox"