feat: |Worker| add REMOVE_ALL_ATTACHMENT and REMOVE_EXCEED_SIZE_ATTAC… (#562)

feat: |Worker| add REMOVE_ALL_ATTACHMENT and REMOVE_EXCEED_SIZE_ATTACHMENT

CHANGELOG.md

@@ -4,6 +4,9 @@
 ## main(v0.8.5)
 
 - feat: |mail-parser-wasm-worker| 修复 `initSync` 函数调用时的 `deprecated` 参数警告
+- feat: rpc headers covert & typo (#559)
+- fix: telegram mail page use iframe show email (#561)
+- feat: |Worker| 增加 `REMOVE_ALL_ATTACHMENT` 和 `REMOVE_EXCEED_SIZE_ATTACHMENT` 用于移除邮件附件，由于是解析邮件的一些信息会丢失，比如图片等.
 
 ## v0.8.4
 

vitepress-docs/docs/en/cli.md

@@ -138,6 +138,10 @@ ENABLE_AUTO_REPLY = false
 # JUNK_MAIL_CHECK_LIST = = ["spf", "dkim", "dmarc"]
 # junk mail force check pass list, if no status or status is not pass, will be marked as junk mail
 # JUNK_MAIL_FORCE_PASS_LIST = ["spf", "dkim", "dmarc"]
+# remove attachment if size exceed 2MB, mail maybe mising some information due to parsing
+# REMOVE_EXCEED_SIZE_ATTACHMENT = true
+# remove all attachment, mail maybe mising some information due to parsing
+# REMOVE_ALL_ATTACHMENT = true
 
 [[d1_databases]]
 binding = "DB"

vitepress-docs/docs/zh/guide/cli/worker.md

@@ -110,6 +110,10 @@ ENABLE_AUTO_REPLY = false
 # JUNK_MAIL_CHECK_LIST = = ["spf", "dkim", "dmarc"]
 # 垃圾邮件检查配置, 任何一项 不存在 或者 不通过 则被判定为垃圾邮件
 # JUNK_MAIL_FORCE_PASS_LIST = ["spf", "dkim", "dmarc"]
+# 如果附件大小超过 2MB，则删除附件，邮件可能由于解析而丢失一些信息
+# REMOVE_EXCEED_SIZE_ATTACHMENT = true
+# 移除所有附件，邮件可能由于解析而丢失一些信息
+# REMOVE_ALL_ATTACHMENT = true
 # 是否开启其他 worker 处理邮件
 # ENABLE_ANOTHER_WORKER = false
 # 其他 worker 处理邮件的配置，可以配置多个其他 worker。

worker/src/admin_api/worker_config.ts

@@ -46,6 +46,9 @@ export default {
             "JUNK_MAIL_CHECK_LIST": getStringArray(c.env.JUNK_MAIL_CHECK_LIST),
             "JUNK_MAIL_FORCE_PASS_LIST": getStringArray(c.env.JUNK_MAIL_FORCE_PASS_LIST),
 
+            "REMOVE_EXCEED_SIZE_ATTACHMENT": getBooleanValue(c.env.REMOVE_EXCEED_SIZE_ATTACHMENT),
+            "REMOVE_ALL_ATTACHMENT": getBooleanValue(c.env.REMOVE_ALL_ATTACHMENT),
+
             "ENABLE_ANOTHER_WORKER": getBooleanValue(c.env.ENABLE_ANOTHER_WORKER),
             "ANOTHER_WORKER_LIST": getAnotherWorkerList(c),
         })

worker/src/email/check_attachment.ts

@@ -0,0 +1,48 @@
+import { Bindings, ParsedEmailContext } from "../types";
+import { getBooleanValue } from "../utils";
+import { commonParseMail } from "../common";
+import { createMimeMessage } from "mimetext";
+
+export const remove_attachment_if_need = async (
+    env: Bindings,
+    parsedEmailContext: ParsedEmailContext,
+    from_address: string,
+    to_address: string,
+    size: number
+): Promise<void> => {
+    // if configured, remove all attachment
+    const removeAllAttachment = getBooleanValue(env.REMOVE_ALL_ATTACHMENT);
+    // if attachment size > 2MB, remove attachment
+    const removeExceedSizeAttachment = getBooleanValue(env.REMOVE_EXCEED_SIZE_ATTACHMENT) && size >= 2 * 1024 * 1024;
+    const shouldRemoveAttachment = removeAllAttachment || removeExceedSizeAttachment;
+    if (!shouldRemoveAttachment) return;
+
+    const parsedEmail = await commonParseMail(parsedEmailContext);
+    if (!parsedEmail) return;
+
+    const msg = createMimeMessage();
+    if (parsedEmail?.headers) {
+        for (const header of parsedEmail.headers) {
+            msg.setHeader(header["key"], header["value"]);
+        }
+    }
+    msg.setSender({
+        name: parsedEmail?.sender || from_address,
+        addr: from_address
+    });
+    msg.setRecipient(to_address);
+    msg.setSubject(parsedEmail?.subject || "Failed to parse email subject");
+    if (parsedEmail?.html) {
+        msg.addMessage({
+            contentType: 'text/html',
+            data: parsedEmail.html
+        });
+    }
+    if (parsedEmail?.text) {
+        msg.addMessage({
+            contentType: 'text/plain',
+            data: parsedEmail.text
+        });
+    }
+    parsedEmailContext.rawEmail = msg.asRaw();
+}

worker/src/email/index.ts

@@ -7,6 +7,7 @@ import { auto_reply } from "./auto_reply";
 import { isBlocked } from "./black_list";
 import { triggerWebhook, triggerAnotherWorker, commonParseMail } from "../common";
 import { check_if_junk_mail } from "./check_junk";
+import { remove_attachment_if_need } from "./check_attachment";
 
 
 async function email(message: ForwardableEmailMessage, env: Bindings, ctx: ExecutionContext) {
@@ -32,13 +33,20 @@ async function email(message: ForwardableEmailMessage, env: Bindings, ctx: Execu
         console.log("check junk mail error", error);
     }
 
+    // remove attachment if configured or size > 2MB
+    try {
+        await remove_attachment_if_need(env, parsedEmailContext, message.from, message.to, message.rawSize);
+    } catch (error) {
+        console.log("remove attachment error", error);
+    }
+
     const message_id = message.headers.get("Message-ID");
     // save email
     try {
         const { success } = await env.DB.prepare(
             `INSERT INTO raw_mails (source, address, raw, message_id) VALUES (?, ?, ?, ?)`
         ).bind(
-            message.from, message.to, rawEmail, message_id
+            message.from, message.to, parsedEmailContext.rawEmail, message_id
         ).run();
         if (!success) {
             message.setReject(`Failed save message to ${message.to}`);

worker/src/types.d.ts

@@ -50,6 +50,8 @@ export type Bindings = {
     ENABLE_ANOTHER_WORKER: string | boolean | undefined
     ANOTHER_WORKER_LIST: string | AnotherWorker[] | undefined
 
+    REMOVE_ALL_ATTACHMENT: string | boolean | undefined
+    REMOVE_EXCEED_SIZE_ATTACHMENT: string | boolean | undefined
 
     // s3 config
     S3_ENDPOINT: string | undefined

worker/wrangler.toml.template

@@ -80,6 +80,10 @@ ENABLE_AUTO_REPLY = false
 # JUNK_MAIL_CHECK_LIST = = ["spf", "dkim", "dmarc"]
 # junk mail force check pass list, if no status or status is not pass, will be marked as junk mail
 # JUNK_MAIL_FORCE_PASS_LIST = ["spf", "dkim", "dmarc"]
+# remove attachment if size exceed 2MB, mail maybe mising some information due to parsing
+# REMOVE_EXCEED_SIZE_ATTACHMENT = true
+# remove all attachment, mail maybe mising some information due to parsing
+# REMOVE_ALL_ATTACHMENT = true
 # Calling other woker to process email
 #ENABLE_ANOTHER_WORKER = false
 #ANOTHER_WORKER_LIST ="""