diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d2756c1..98c539c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ ### Bug Fixes +- fix: |Worker| 按邮件认证规范修复垃圾邮件检测：SPF、DKIM、DMARC 的 `none` 及 SPF/DKIM `neutral` 按认证方法不存在处理，并忽略未注册结果和不支持的方法版本；`JUNK_MAIL_FORCE_PASS_LIST` 仍要求明确返回受支持的 `pass` - fix: |Admin| 管理后台删除邮箱地址时，先删除该地址的邮件、发件记录、自动回复等关联数据，最后再删除地址本身；此前地址行先被删除导致按地址名匹配的子查询查不到数据，邮件等记录被遗留在数据库中 - fix: |AI 提取| 强化提示词，要求 AI 保持邮件原始链接域名，避免小模型改写验证链接域名导致错误跳转（issue #1072） - fix: |AI 提取| HTML-only 邮件在发送给 Workers AI 前会先压缩为可读文本，避免样式模板过长导致验证码位于 4000 字截断之后而无法识别 diff --git a/CHANGELOG_EN.md b/CHANGELOG_EN.md index 397d8a5a..7fc5a421 100644 --- a/CHANGELOG_EN.md +++ b/CHANGELOG_EN.md @@ -15,6 +15,7 @@ ### Bug Fixes +- fix: |Worker| Align junk-mail checking with authentication standards: treat SPF, DKIM, and DMARC `none` plus SPF/DKIM `neutral` as absent, and ignore unregistered results and unsupported method versions; `JUNK_MAIL_FORCE_PASS_LIST` still requires an explicit supported `pass` - fix: |Admin| When deleting an address from the admin panel, delete its mails, sender records, sendbox and auto-reply entries before removing the address row itself; previously the address row was deleted first, so the name-based subqueries matched nothing and the mails were left orphaned in the database - fix: |AI Extract| Strengthen the prompt to keep original link domains from the email, preventing small models from rewriting verification-link domains (issue #1072) - fix: |AI Extract| Convert HTML-only mail bodies into compact readable text before sending them to Workers AI, preventing long templates from pushing verification codes past the 4000-character truncation window diff --git a/vitepress-docs/docs/en/cli.md b/vitepress-docs/docs/en/cli.md index 9f33be71..79ef0196 100644 --- a/vitepress-docs/docs/en/cli.md +++ b/vitepress-docs/docs/en/cli.md @@ -141,9 +141,9 @@ ENABLE_AUTO_REPLY = false # FRONTEND_URL = "https://xxxx.xxx" # Enable check junk mail # ENABLE_CHECK_JUNK_MAIL = false -# junk mail check list, if status exists and status is not pass, will be marked as junk mail -# JUNK_MAIL_CHECK_LIST = = ["spf", "dkim", "dmarc"] -# junk mail force check pass list, if no status or status is not pass, will be marked as junk mail +# junk mail check list: reject registered failure/error results; none and SPF/DKIM neutral are treated as absent +# JUNK_MAIL_CHECK_LIST = ["spf", "dkim", "dmarc"] +# junk mail force pass list: every configured method must explicitly return pass # JUNK_MAIL_FORCE_PASS_LIST = ["spf", "dkim", "dmarc"] # remove attachment if size exceed 2MB, mail maybe mising some information due to parsing # REMOVE_EXCEED_SIZE_ATTACHMENT = true diff --git a/vitepress-docs/docs/en/guide/worker-vars.md b/vitepress-docs/docs/en/guide/worker-vars.md index 508bef1c..c1f85ff4 100644 --- a/vitepress-docs/docs/en/guide/worker-vars.md +++ b/vitepress-docs/docs/en/guide/worker-vars.md @@ -97,14 +97,16 @@ | ------------------------------- | --------- | ---------------------------------------------------------------------------------------------------------------------- | -------------------------- | | `BLACK_LIST` | Text | Blacklist for filtering senders, comma separated | `gov.cn,edu.cn` | | `ENABLE_CHECK_JUNK_MAIL` | Text/JSON | Whether to enable junk mail checking, used with the following two lists | `false` | -| `JUNK_MAIL_CHECK_LIST` | JSON | Junk mail check configuration, marked as junk if any item `exists` and `fails` | `["spf", "dkim", "dmarc"]` | -| `JUNK_MAIL_FORCE_PASS_LIST` | JSON | Junk mail check configuration, marked as junk if any item `does not exist` or `fails` | `["spf", "dkim", "dmarc"]` | +| `JUNK_MAIL_CHECK_LIST` | JSON | Existence check; registered failure/error results are junk, while `none` and SPF/DKIM `neutral` are treated as absent | `["spf", "dkim", "dmarc"]` | +| `JUNK_MAIL_FORCE_PASS_LIST` | JSON | Strict pass check; every item must explicitly return `pass`, otherwise it is treated as junk | `["spf", "dkim", "dmarc"]` | | `FORWARD_ADDRESS_LIST` | JSON | Global forward address list, disabled if not configured, all emails will be forwarded to listed addresses when enabled | `["xxx@xxx.com"]` | | `REMOVE_EXCEED_SIZE_ATTACHMENT` | Text/JSON | If attachment exceeds 2MB, remove it, email may lose some information due to parsing | `true` | | `REMOVE_ALL_ATTACHMENT` | Text/JSON | Remove all attachments, email may lose some information due to parsing | `true` | | `ENABLE_MAIL_GZIP` | Text/JSON | When enabled, new emails are gzip-compressed and stored in `raw_blob` column to save D1 database space. Existing plaintext `raw` data is automatically compatible for reading. **Run database migration first (`Admin -> Quick Setup -> Database -> Migrate Database` or `POST /admin/db_migration`) to ensure the `raw_blob` column exists before enabling. This feature adds compression/decompression CPU overhead, so enabling it on a paid Cloudflare Worker plan is recommended.** | `true` | > [!NOTE] +> Authentication results follow their standards: SPF `none` means no usable domain or SPF record was found, and SPF `neutral` must be treated like `none`; DKIM `none` means the message was unsigned, and DKIM `neutral` is also treated as unsigned; DMARC `none` means no applicable DMARC policy was found. Unregistered results and unsupported method versions are ignored. `JUNK_MAIL_CHECK_LIST` treats these results as absent, while `JUNK_MAIL_FORCE_PASS_LIST` still requires an explicit supported `pass` +> > `ENABLE_MAIL_GZIP` adds CPU cost for gzip compression on write and decompression on read. Free-tier Workers are more likely to hit CPU limits, so a paid plan is recommended before enabling it > > `Junk mail checking` and `attachment removal` require email parsing, free tier CPU is limited, may cause large email parsing timeout diff --git a/vitepress-docs/docs/zh/guide/worker-vars.md b/vitepress-docs/docs/zh/guide/worker-vars.md index 98211710..9f96bd9b 100644 --- a/vitepress-docs/docs/zh/guide/worker-vars.md +++ b/vitepress-docs/docs/zh/guide/worker-vars.md @@ -91,14 +91,16 @@ | ------------------------------- | --------- | -------------------------------------------------------------------------- | -------------------------- | | `BLACK_LIST` | 文本 | 黑名单，用于过滤发件人，逗号分隔 | `gov.cn,edu.cn` | | `ENABLE_CHECK_JUNK_MAIL` | 文本/JSON | 是否启用垃圾邮件检查，配合下列两个列表使用 | `false` | -| `JUNK_MAIL_CHECK_LIST` | JSON | 垃圾邮件检查配置, 任何一项 `存在` 且 `不通过` 则被判定为垃圾邮件 | `["spf", "dkim", "dmarc"]` | -| `JUNK_MAIL_FORCE_PASS_LIST` | JSON | 垃圾邮件检查配置, 任何一项 `不存在` 或者 `不通过` 则被判定为垃圾邮件 | `["spf", "dkim", "dmarc"]` | +| `JUNK_MAIL_CHECK_LIST` | JSON | 存在性检查；已注册的失败/错误结果判定为垃圾邮件，`none` 和 SPF/DKIM `neutral` 按不存在处理 | `["spf", "dkim", "dmarc"]` | +| `JUNK_MAIL_FORCE_PASS_LIST` | JSON | 强制通过检查；每一项都必须明确返回 `pass`，否则判定为垃圾邮件 | `["spf", "dkim", "dmarc"]` | | `FORWARD_ADDRESS_LIST` | JSON | 全局转发地址列表，如果不配置则不启用，启用后所有邮件都会转发到列表中的地址 | `["xxx@xxx.com"]` | | `REMOVE_EXCEED_SIZE_ATTACHMENT` | 文本/JSON | 如果附件大小超过 2MB，则删除附件，邮件可能由于解析而丢失一些信息 | `true` | | `REMOVE_ALL_ATTACHMENT` | 文本/JSON | 移除所有附件，邮件可能由于解析而丢失一些信息 | `true` | | `ENABLE_MAIL_GZIP` | 文本/JSON | 启用后新邮件将 Gzip 压缩存储到 `raw_blob` 字段，可节省 D1 数据库空间。已有明文 `raw` 数据自动兼容读取。**启用前请先执行数据库迁移（`Admin -> 快速设置 -> 数据库 -> 升级数据库 Schema` 或 `POST /admin/db_migration`），确保 `raw_blob` 列已创建。该功能会增加压缩/解压 CPU 开销，建议使用 Cloudflare Worker 付费 Plan 再开启。** | `true` | > [!NOTE] +> 认证结果遵循各自规范：SPF `none` 表示没有可检查的域名或 SPF 记录，SPF `neutral` 必须与 `none` 相同处理；DKIM `none` 表示邮件未签名，DKIM `neutral` 同样按未签名处理；DMARC `none` 表示没有适用的 DMARC 策略。未注册结果和不支持的方法版本也会被忽略。`JUNK_MAIL_CHECK_LIST` 将这些结果视为认证方法不存在，`JUNK_MAIL_FORCE_PASS_LIST` 仍只接受明确且受支持的 `pass` +> > `ENABLE_MAIL_GZIP` 会增加邮件写入压缩与读取解压的 CPU 消耗，免费版 Worker 更容易触发 CPU 限制，建议付费 Plan 再开启 > > `垃圾邮件检查` 和 `移除附件功能` 需要解析邮件，免费版 CPU 有限，可能会导致大邮件解析超时 diff --git a/worker/src/email/check_junk.ts b/worker/src/email/check_junk.ts index af2b59f2..2fe9cb53 100644 --- a/worker/src/email/check_junk.ts +++ b/worker/src/email/check_junk.ts @@ -1,5 +1,6 @@ import { getBooleanValue, getStringArray } from "../utils"; import { commonParseMail } from "../common"; +import { isJunkMailByHeaders } from "./junk_mail_policy"; export const check_if_junk_mail = async ( env: Bindings, address: string, @@ -14,50 +15,9 @@ export const check_if_junk_mail = async ( const checkListWhenExist = getStringArray(env.JUNK_MAIL_CHECK_LIST); const forcePassList = getStringArray(env.JUNK_MAIL_FORCE_PASS_LIST); - const passedList: string[] = []; - const existList: string[] = []; - - const headers = parsedEmail.headers; - for (const header of headers) { - if (!header["key"]) continue; - if (!header["value"]) continue; - - // check spf - if (header["key"].toLowerCase() == "received-spf") { - existList.push("spf"); - if (header["value"].toLowerCase().includes("pass")) { - passedList.push("spf"); - } - } - - // check dkim and dmarc - if (header["key"].toLowerCase() == "authentication-results") { - if (header["value"].toLowerCase().includes("dkim=")) { - existList.push("dkim"); - if (header["value"].toLowerCase().includes("dkim=pass")) { - passedList.push("dkim"); - } - } - if (header["value"].toLowerCase().includes("dmarc=")) { - existList.push("dmarc"); - if (header["value"].toLowerCase().includes("dmarc=pass")) { - passedList.push("dmarc"); - } - } - } - } - // check if all checkListWhenExist item passed when exist - if (checkListWhenExist?.some( - (checkName) => existList.includes(checkName.toLowerCase()) - && !passedList.includes(checkName.toLowerCase()) - )) { - return true; - } - - if (forcePassList?.length == 0) return false; - - // check force pass list - return forcePassList.some( - (checkName) => !passedList.includes(checkName.toLowerCase()) + return isJunkMailByHeaders( + parsedEmail.headers, + checkListWhenExist, + forcePassList, ); } diff --git a/worker/src/email/junk_mail_policy.ts b/worker/src/email/junk_mail_policy.ts new file mode 100644 index 00000000..a1f18345 --- /dev/null +++ b/worker/src/email/junk_mail_policy.ts @@ -0,0 +1,88 @@ +type AuthenticationResult = { + method: string, + result: string, + version?: string, +} + +const supportedResults: Record> = { + spf: new Set(["none", "neutral", "pass", "fail", "softfail", "policy", "temperror", "permerror"]), + dkim: new Set(["none", "neutral", "pass", "fail", "policy", "temperror", "permerror"]), + dmarc: new Set(["none", "pass", "fail", "temperror", "permerror"]), +}; +const receivedSpfResults = new Set([ + "none", "neutral", "pass", "fail", "softfail", "temperror", "permerror", +]); + +const parseAuthenticationResults = (value: string): AuthenticationResult[] => { + const results: AuthenticationResult[] = []; + const pattern = /(?:^|;)[ \t]*([a-z][a-z0-9_-]*)(?:[ \t]*\/[ \t]*(\d+))?[ \t]*=[ \t]*([a-z][a-z0-9_-]*)/gi; + + for (const match of value.matchAll(pattern)) { + results.push({ + method: match[1].toLowerCase(), + version: match[2], + result: match[3].toLowerCase(), + }); + } + return results; +} + +const normalizeCheckName = (value: string): string => value.trim().toLowerCase(); + +const trackAuthenticationResult = ( + method: string, + result: string | undefined, + existing: Set, + passed: Set, +) => { + if (!result || !supportedResults[method]?.has(result)) return; + + // SPF and DKIM neutral are absent-equivalent under RFC 7208 and RFC 8601. + if (result === "none" || ((method === "spf" || method === "dkim") && result === "neutral")) return; + + existing.add(method); + if (result === "pass") { + passed.add(method); + } +} + +export const isJunkMailByHeaders = ( + headers: Record[], + checkListWhenExist: string[], + forcePassList: string[], +): boolean => { + const passed = new Set(); + const existing = new Set(); + + for (const header of headers) { + const key = header["key"]?.toLowerCase(); + const value = header["value"]; + if (!key || !value) continue; + + if (key === "received-spf") { + const result = value.match(/^[ \t]*([a-z][a-z0-9_-]*)/i)?.[1]?.toLowerCase(); + if (!result || !receivedSpfResults.has(result)) continue; + trackAuthenticationResult("spf", result, existing, passed); + continue; + } + + if (key !== "authentication-results") continue; + + for (const { method, result, version } of parseAuthenticationResults(value)) { + if (method !== "spf" && method !== "dkim" && method !== "dmarc") continue; + if (version && version !== "1") continue; + trackAuthenticationResult(method, result, existing, passed); + } + } + + if (checkListWhenExist.some((checkName) => { + const normalizedName = normalizeCheckName(checkName); + return existing.has(normalizedName) && !passed.has(normalizedName); + })) { + return true; + } + + return forcePassList.some( + (checkName) => !passed.has(normalizeCheckName(checkName)) + ); +} diff --git a/worker/wrangler.toml.template b/worker/wrangler.toml.template index 62b47586..01096517 100644 --- a/worker/wrangler.toml.template +++ b/worker/wrangler.toml.template @@ -126,9 +126,9 @@ ENABLE_AUTO_REPLY = false # FRONTEND_URL = "https://xxxx.xxx" # Enable check junk mail # ENABLE_CHECK_JUNK_MAIL = false -# junk mail check list, if status exists and status is not pass, will be marked as junk mail -# JUNK_MAIL_CHECK_LIST = = ["spf", "dkim", "dmarc"] -# junk mail force check pass list, if no status or status is not pass, will be marked as junk mail +# junk mail check list: reject registered failure/error results; none and SPF/DKIM neutral are treated as absent +# JUNK_MAIL_CHECK_LIST = ["spf", "dkim", "dmarc"] +# junk mail force pass list: every configured method must explicitly return pass # JUNK_MAIL_FORCE_PASS_LIST = ["spf", "dkim", "dmarc"] # remove attachment if size exceed 2MB, mail maybe mising some information due to parsing # REMOVE_EXCEED_SIZE_ATTACHMENT = true