From 047200c1c22a0d8e93166cd9c8b511f765bc3e67 Mon Sep 17 00:00:00 2001 From: Dream Hunter Date: Sat, 18 Jan 2025 14:12:01 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20|Worker|=20add=20REMOVE=5FALL=5FATTACHM?= =?UTF-8?q?ENT=20and=20REMOVE=5FEXCEED=5FSIZE=5FATTAC=E2=80=A6=20(#562)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit feat: |Worker| add REMOVE_ALL_ATTACHMENT and REMOVE_EXCEED_SIZE_ATTACHMENT --- CHANGELOG.md | 3 ++ vitepress-docs/docs/en/cli.md | 4 ++ vitepress-docs/docs/zh/guide/cli/worker.md | 4 ++ worker/src/admin_api/worker_config.ts | 3 ++ worker/src/email/check_attachment.ts | 48 ++++++++++++++++++++++ worker/src/email/index.ts | 10 ++++- worker/src/types.d.ts | 2 + worker/wrangler.toml.template | 4 ++ 8 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 worker/src/email/check_attachment.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index b6eaa6aa..b8dfc805 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ ## main(v0.8.5) - feat: |mail-parser-wasm-worker| 修复 `initSync` 函数调用时的 `deprecated` 参数警告 +- feat: rpc headers covert & typo (#559) +- fix: telegram mail page use iframe show email (#561) +- feat: |Worker| 增加 `REMOVE_ALL_ATTACHMENT` 和 `REMOVE_EXCEED_SIZE_ATTACHMENT` 用于移除邮件附件，由于是解析邮件的一些信息会丢失，比如图片等. ## v0.8.4 diff --git a/vitepress-docs/docs/en/cli.md b/vitepress-docs/docs/en/cli.md index b32413a7..acc9f210 100644 --- a/vitepress-docs/docs/en/cli.md +++ b/vitepress-docs/docs/en/cli.md @@ -138,6 +138,10 @@ ENABLE_AUTO_REPLY = false # JUNK_MAIL_CHECK_LIST = = ["spf", "dkim", "dmarc"] # junk mail force check pass list, if no status or status is not pass, will be marked as junk mail # JUNK_MAIL_FORCE_PASS_LIST = ["spf", "dkim", "dmarc"] +# remove attachment if size exceed 2MB, mail maybe mising some information due to parsing +# REMOVE_EXCEED_SIZE_ATTACHMENT = true +# remove all attachment, mail maybe mising some information due to parsing +# REMOVE_ALL_ATTACHMENT = true [[d1_databases]] binding = "DB" diff --git a/vitepress-docs/docs/zh/guide/cli/worker.md b/vitepress-docs/docs/zh/guide/cli/worker.md index b2bfc022..b94045ed 100644 --- a/vitepress-docs/docs/zh/guide/cli/worker.md +++ b/vitepress-docs/docs/zh/guide/cli/worker.md @@ -110,6 +110,10 @@ ENABLE_AUTO_REPLY = false # JUNK_MAIL_CHECK_LIST = = ["spf", "dkim", "dmarc"] # 垃圾邮件检查配置, 任何一项 不存在 或者 不通过 则被判定为垃圾邮件 # JUNK_MAIL_FORCE_PASS_LIST = ["spf", "dkim", "dmarc"] +# 如果附件大小超过 2MB，则删除附件，邮件可能由于解析而丢失一些信息 +# REMOVE_EXCEED_SIZE_ATTACHMENT = true +# 移除所有附件，邮件可能由于解析而丢失一些信息 +# REMOVE_ALL_ATTACHMENT = true # 是否开启其他 worker 处理邮件 # ENABLE_ANOTHER_WORKER = false # 其他 worker 处理邮件的配置，可以配置多个其他 worker。 diff --git a/worker/src/admin_api/worker_config.ts b/worker/src/admin_api/worker_config.ts index 14b9af5b..eda1487c 100644 --- a/worker/src/admin_api/worker_config.ts +++ b/worker/src/admin_api/worker_config.ts @@ -46,6 +46,9 @@ export default { "JUNK_MAIL_CHECK_LIST": getStringArray(c.env.JUNK_MAIL_CHECK_LIST), "JUNK_MAIL_FORCE_PASS_LIST": getStringArray(c.env.JUNK_MAIL_FORCE_PASS_LIST), + "REMOVE_EXCEED_SIZE_ATTACHMENT": getBooleanValue(c.env.REMOVE_EXCEED_SIZE_ATTACHMENT), + "REMOVE_ALL_ATTACHMENT": getBooleanValue(c.env.REMOVE_ALL_ATTACHMENT), + "ENABLE_ANOTHER_WORKER": getBooleanValue(c.env.ENABLE_ANOTHER_WORKER), "ANOTHER_WORKER_LIST": getAnotherWorkerList(c), }) diff --git a/worker/src/email/check_attachment.ts b/worker/src/email/check_attachment.ts new file mode 100644 index 00000000..72055ea6 --- /dev/null +++ b/worker/src/email/check_attachment.ts @@ -0,0 +1,48 @@ +import { Bindings, ParsedEmailContext } from "../types"; +import { getBooleanValue } from "../utils"; +import { commonParseMail } from "../common"; +import { createMimeMessage } from "mimetext"; + +export const remove_attachment_if_need = async ( + env: Bindings, + parsedEmailContext: ParsedEmailContext, + from_address: string, + to_address: string, + size: number +): Promise => { + // if configured, remove all attachment + const removeAllAttachment = getBooleanValue(env.REMOVE_ALL_ATTACHMENT); + // if attachment size > 2MB, remove attachment + const removeExceedSizeAttachment = getBooleanValue(env.REMOVE_EXCEED_SIZE_ATTACHMENT) && size >= 2 * 1024 * 1024; + const shouldRemoveAttachment = removeAllAttachment || removeExceedSizeAttachment; + if (!shouldRemoveAttachment) return; + + const parsedEmail = await commonParseMail(parsedEmailContext); + if (!parsedEmail) return; + + const msg = createMimeMessage(); + if (parsedEmail?.headers) { + for (const header of parsedEmail.headers) { + msg.setHeader(header["key"], header["value"]); + } + } + msg.setSender({ + name: parsedEmail?.sender || from_address, + addr: from_address + }); + msg.setRecipient(to_address); + msg.setSubject(parsedEmail?.subject || "Failed to parse email subject"); + if (parsedEmail?.html) { + msg.addMessage({ + contentType: 'text/html', + data: parsedEmail.html + }); + } + if (parsedEmail?.text) { + msg.addMessage({ + contentType: 'text/plain', + data: parsedEmail.text + }); + } + parsedEmailContext.rawEmail = msg.asRaw(); +} diff --git a/worker/src/email/index.ts b/worker/src/email/index.ts index 0ea80f52..0894d10c 100644 --- a/worker/src/email/index.ts +++ b/worker/src/email/index.ts @@ -7,6 +7,7 @@ import { auto_reply } from "./auto_reply"; import { isBlocked } from "./black_list"; import { triggerWebhook, triggerAnotherWorker, commonParseMail } from "../common"; import { check_if_junk_mail } from "./check_junk"; +import { remove_attachment_if_need } from "./check_attachment"; async function email(message: ForwardableEmailMessage, env: Bindings, ctx: ExecutionContext) { @@ -32,13 +33,20 @@ async function email(message: ForwardableEmailMessage, env: Bindings, ctx: Execu console.log("check junk mail error", error); } + // remove attachment if configured or size > 2MB + try { + await remove_attachment_if_need(env, parsedEmailContext, message.from, message.to, message.rawSize); + } catch (error) { + console.log("remove attachment error", error); + } + const message_id = message.headers.get("Message-ID"); // save email try { const { success } = await env.DB.prepare( `INSERT INTO raw_mails (source, address, raw, message_id) VALUES (?, ?, ?, ?)` ).bind( - message.from, message.to, rawEmail, message_id + message.from, message.to, parsedEmailContext.rawEmail, message_id ).run(); if (!success) { message.setReject(`Failed save message to ${message.to}`); diff --git a/worker/src/types.d.ts b/worker/src/types.d.ts index 703763cd..4eba49e7 100644 --- a/worker/src/types.d.ts +++ b/worker/src/types.d.ts @@ -50,6 +50,8 @@ export type Bindings = { ENABLE_ANOTHER_WORKER: string | boolean | undefined ANOTHER_WORKER_LIST: string | AnotherWorker[] | undefined + REMOVE_ALL_ATTACHMENT: string | boolean | undefined + REMOVE_EXCEED_SIZE_ATTACHMENT: string | boolean | undefined // s3 config S3_ENDPOINT: string | undefined diff --git a/worker/wrangler.toml.template b/worker/wrangler.toml.template index 464f8063..1817a7b1 100644 --- a/worker/wrangler.toml.template +++ b/worker/wrangler.toml.template @@ -80,6 +80,10 @@ ENABLE_AUTO_REPLY = false # JUNK_MAIL_CHECK_LIST = = ["spf", "dkim", "dmarc"] # junk mail force check pass list, if no status or status is not pass, will be marked as junk mail # JUNK_MAIL_FORCE_PASS_LIST = ["spf", "dkim", "dmarc"] +# remove attachment if size exceed 2MB, mail maybe mising some information due to parsing +# REMOVE_EXCEED_SIZE_ATTACHMENT = true +# remove all attachment, mail maybe mising some information due to parsing +# REMOVE_ALL_ATTACHMENT = true # Calling other woker to process email #ENABLE_ANOTHER_WORKER = false #ANOTHER_WORKER_LIST ="""