feat: add JUNK_MAIL_CHECK_LIST for check exits and passed item && add ParsedEmailContext to cache the parsed Email (#553)

* feat: Junk mail only check JUNK_MAIL_FORCE_PASS_LIST

* feat: add `JUNK_MAIL_CHECK_LIST` for check exits and passed item && add `ParsedEmailContext` to cache the parsed Email

CHANGELOG.md

@@ -8,6 +8,8 @@
 - feat: 增加 worker 配置 `DISABLE_ANONYMOUS_USER_CREATE_EMAIL` 禁用匿名用户创建邮箱地址，只允许登录用户创建邮箱地址
 - feat: 增加 worker 配置 `ENABLE_ANOTHER_WORKER` 及 `ANOTHER_WORKER_LIST` ，用于调用其他 worker 的 rpc 接口 (#547)
 - feat: |UI| 自动刷新配置保存到浏览器，可配置刷新间隔
+- feat: 垃圾邮件检测增加存在是才检查的列表 `JUNK_MAIL_CHECK_LIST` 配置
+- feat: | Worker | 增加 `ParsedEmailContext` 类用于缓存解析后的邮件内容，减少解析次数
 
 ## v0.8.3
 

vitepress-docs/docs/en/cli.md

@@ -134,6 +134,8 @@ ENABLE_AUTO_REPLY = false
 # FRONTEND_URL = "https://xxxx.xxx"
 # Enable check junk mail
 # ENABLE_CHECK_JUNK_MAIL = false
+# junk mail check list, if status exists and status is not pass, will be marked as junk mail
+# JUNK_MAIL_CHECK_LIST = = ["spf", "dkim", "dmarc"]
 # junk mail force check pass list, if no status or status is not pass, will be marked as junk mail
 # JUNK_MAIL_FORCE_PASS_LIST = ["spf", "dkim", "dmarc"]
 

vitepress-docs/docs/zh/guide/cli/worker.md

@@ -106,7 +106,9 @@ ENABLE_AUTO_REPLY = false
 # FRONTEND_URL = "https://xxxx.xxx"
 # 是否启用垃圾邮件检查，默认任何一项存在配置且不通过则被判定为垃圾邮件
 # ENABLE_CHECK_JUNK_MAIL = false
-# 垃圾邮件检查配置, 任何一项不存在或者不通过则被判定为垃圾邮件
+# 垃圾邮件检查配置, 任何一项 存在 且 不通过 则被判定为垃圾邮件
+# JUNK_MAIL_CHECK_LIST = = ["spf", "dkim", "dmarc"]
+# 垃圾邮件检查配置, 任何一项 不存在 或者 不通过 则被判定为垃圾邮件
 # JUNK_MAIL_FORCE_PASS_LIST = ["spf", "dkim", "dmarc"]
 # 是否开启其他 worker 处理邮件
 # ENABLE_ANOTHER_WORKER = false

worker/src/admin_api/mail_webhook_settings.ts

@@ -1,5 +1,5 @@
 import { Context } from "hono";
-import { HonoCustomType } from "../types";
+import { HonoCustomType, ParsedEmailContext } from "../types";
 import { CONSTANTS } from "../constants";
 import { WebhookSettings } from "../models";
 import { commonParseMail, sendWebhook } from "../common";
@@ -25,8 +25,8 @@ async function testWebhookSettings(c: Context<HonoCustomType>): Promise<Response
     const { id: mailId, raw } = await c.env.DB.prepare(
         `SELECT id, raw FROM raw_mails ORDER BY RANDOM() LIMIT 1`
     ).first<{ id: string, raw: string }>() || {};
-
+    const parsedEmailContext: ParsedEmailContext = { rawEmail: raw || "" };
-    const parsedEmail = await commonParseMail(raw);
+    const parsedEmail = await commonParseMail(parsedEmailContext);
     const res = await sendWebhook(settings, {
         id: mailId || "0",
         url: c.env.FRONTEND_URL ? `${c.env.FRONTEND_URL}?mail_id=${mailId}` : "",

worker/src/admin_api/worker_config.ts

@@ -43,6 +43,7 @@ export default {
             "DISABLE_SHOW_GITHUB": !getBooleanValue(c.env.DISABLE_SHOW_GITHUB),
             "DISABLE_ADMIN_PASSWORD_CHECK": getBooleanValue(c.env.DISABLE_ADMIN_PASSWORD_CHECK),
             "ENABLE_CHECK_JUNK_MAIL": getBooleanValue(c.env.ENABLE_CHECK_JUNK_MAIL),
+            "JUNK_MAIL_CHECK_LIST": getStringArray(c.env.JUNK_MAIL_CHECK_LIST),
             "JUNK_MAIL_FORCE_PASS_LIST": getStringArray(c.env.JUNK_MAIL_FORCE_PASS_LIST),
 
             "ENABLE_ANOTHER_WORKER": getBooleanValue(c.env.ENABLE_ANOTHER_WORKER),

worker/src/common.ts

@@ -2,7 +2,7 @@ import { Context } from 'hono';
 import { Jwt } from 'hono/utils/jwt'
 
 import { getBooleanValue, getDomains, getStringValue, getIntValue, getUserRoles, getDefaultDomains, getJsonSetting, getAnotherWorkerList } from './utils';
-import { HonoCustomType, UserRole, AnotherWorker, RPCEmailMessage } from './types';
+import { HonoCustomType, UserRole, AnotherWorker, RPCEmailMessage, ParsedEmailContext } from './types';
 import { unbindTelegramByAddress } from './telegram_api/common';
 import { CONSTANTS } from './constants';
 import { AdminWebhookSettings, WebhookMail, WebhookSettings } from './models';
@@ -256,16 +256,22 @@ export const handleListQuery = async (
 }
 
 
-export const commonParseMail = async (raw_mail: string | undefined | null): Promise<{
+export const commonParseMail = async (parsedEmailContext: ParsedEmailContext): Promise<{
     sender: string,
     subject: string,
     text: string,
     html: string,
     headers?: Record<string, string>[]
 } | undefined> => {
-    if (!raw_mail) {
+    // check parsed email context is valid
+    if (!parsedEmailContext || !parsedEmailContext.rawEmail) {
         return undefined;
     }
+    // return parsed email if already parsed
+    if (parsedEmailContext.parsedEmail) {
+        return parsedEmailContext.parsedEmail;
+    }
+    const raw_mail = parsedEmailContext.rawEmail;
     // TODO: WASM parse email
     // try {
     //     const { parse_message_wrapper } = await import('mail-parser-wasm-worker');
@@ -275,7 +281,9 @@ export const commonParseMail = async (raw_mail: string | undefined | null): Prom
     //         sender: parsedEmail.sender || "",
     //         subject: parsedEmail.subject || "",
     //         text: parsedEmail.text || "",
-    //         headers: parsedEmail.headers || [],
+    //         headers: parsedEmail.headers?.map(
+    //             (header) => ({ key: header.key, value: header.value })
+    //         ) || [],
     //         html: parsedEmail.body_html || "",
     //     };
     // } catch (e) {
@@ -358,9 +366,9 @@ export async function sendWebhook(settings: WebhookSettings, formatMap: WebhookM
 export async function triggerWebhook(
     c: Context<HonoCustomType>,
     address: string,
-    raw_mail: string,
+    parsedEmailContext: ParsedEmailContext,
     message_id: string | null
-): Promise<string | undefined | null> {
+): Promise<void> {
     if (!c.env.KV || !getBooleanValue(c.env.ENABLE_WEBHOOK)) {
         return
     }
@@ -391,14 +399,14 @@ export async function triggerWebhook(
         `SELECT id FROM raw_mails where address = ? and message_id = ?`
     ).bind(address, message_id).first<string>("id");
 
-    const parsedEmail = await commonParseMail(raw_mail);
+    const parsedEmail = await commonParseMail(parsedEmailContext);
     const webhookMail = {
         id: mailId || "",
         url: c.env.FRONTEND_URL ? `${c.env.FRONTEND_URL}?mail_id=${mailId}` : "",
         from: parsedEmail?.sender || "",
         to: address,
         subject: parsedEmail?.subject || "",
-        raw: raw_mail,
+        raw: parsedEmailContext.rawEmail || "",
         parsedText: parsedEmail?.text || "",
         parsedHtml: parsedEmail?.html || ""
     }
@@ -408,7 +416,6 @@ export async function triggerWebhook(
             console.error(res.message);
         }
     }
-    return webhookMail.parsedText
 }
 
 export async function triggerAnotherWorker(

worker/src/email/check_junk.ts

@@ -1,19 +1,22 @@
-import { Bindings } from "../types";
+import { Bindings, ParsedEmailContext } from "../types";
 import { getBooleanValue, getStringArray } from "../utils";
 import { commonParseMail } from "../common";
 
 export const check_if_junk_mail = async (
     env: Bindings, address: string,
-    raw_mail: string, message_id: string | null
+    parsedEmailContext: ParsedEmailContext,
+    message_id: string | null
 ): Promise<boolean> => {
     if (!getBooleanValue(env.ENABLE_CHECK_JUNK_MAIL)) {
         return false;
     }
-    const parsedEmail = await commonParseMail(raw_mail);
+    const parsedEmail = await commonParseMail(parsedEmailContext);
     if (!parsedEmail?.headers) return false;
 
+    const checkListWhenExist = getStringArray(env.JUNK_MAIL_CHECK_LIST);
     const forcePassList = getStringArray(env.JUNK_MAIL_FORCE_PASS_LIST);
     const passedList: string[] = [];
+    const existList: string[] = [];
 
     const headers = parsedEmail.headers;
     for (const header of headers) {
@@ -22,28 +25,35 @@ export const check_if_junk_mail = async (
 
         // check spf
         if (header["key"].toLowerCase() == "received-spf") {
-            if (!header["value"].toLowerCase().includes("pass")) {
+            existList.push("spf");
-                return true;
+            if (header["value"].toLowerCase().includes("pass")) {
+                passedList.push("spf");
             }
-            passedList.push("spf");
         }
 
         // check dkim and dmarc
         if (header["key"].toLowerCase() == "authentication-results") {
             if (header["value"].toLowerCase().includes("dkim=")) {
-                if (!header["value"].toLowerCase().includes("dkim=pass")) {
+                existList.push("dkim");
-                    return true;
+                if (header["value"].toLowerCase().includes("dkim=pass")) {
+                    passedList.push("dkim");
                 }
-                passedList.push("dkim");
             }
             if (header["value"].toLowerCase().includes("dmarc=")) {
-                if (!header["value"].toLowerCase().includes("dmarc=pass")) {
+                existList.push("dmarc");
-                    return true;
+                if (header["value"].toLowerCase().includes("dmarc=pass")) {
+                    passedList.push("dmarc");
                 }
-                passedList.push("dmarc");
             }
         }
     }
+    // check if all checkListWhenExist item passed when exist
+    if (checkListWhenExist?.some(
+        (checkName) => existList.includes(checkName.toLowerCase())
+            && !passedList.includes(checkName.toLowerCase())
+    )) {
+        return true;
+    }
 
     if (forcePassList?.length == 0) return false;
 

worker/src/email/index.ts

@@ -2,10 +2,10 @@ import { Context } from "hono";
 
 import { getEnvStringList } from "../utils";
 import { sendMailToTelegram } from "../telegram_api";
-import { Bindings, HonoCustomType, RPCEmailMessage } from "../types";
+import { Bindings, HonoCustomType, RPCEmailMessage, ParsedEmailContext } from "../types";
 import { auto_reply } from "./auto_reply";
 import { isBlocked } from "./black_list";
-import { triggerWebhook, triggerAnotherWorker, commonParseMail} from "../common";
+import { triggerWebhook, triggerAnotherWorker, commonParseMail } from "../common";
 import { check_if_junk_mail } from "./check_junk";
 
 
@@ -16,10 +16,13 @@ async function email(message: ForwardableEmailMessage, env: Bindings, ctx: Execu
         return;
     }
     const rawEmail = await new Response(message.raw).text();
+    const parsedEmailContext: ParsedEmailContext = {
+        rawEmail: rawEmail
+    };
 
     // check if junk mail
     try {
-        const is_junk = await check_if_junk_mail(env, message.to, rawEmail, message.headers.get("Message-ID"));
+        const is_junk = await check_if_junk_mail(env, message.to, parsedEmailContext, message.headers.get("Message-ID"));
         if (is_junk) {
             message.setReject("Junk mail");
             console.log(`Junk mail from ${message.from} to ${message.to}`);
@@ -31,14 +34,19 @@ async function email(message: ForwardableEmailMessage, env: Bindings, ctx: Execu
 
     const message_id = message.headers.get("Message-ID");
     // save email
-    const { success } = await env.DB.prepare(
+    try {
-        `INSERT INTO raw_mails (source, address, raw, message_id) VALUES (?, ?, ?, ?)`
+        const { success } = await env.DB.prepare(
-    ).bind(
+            `INSERT INTO raw_mails (source, address, raw, message_id) VALUES (?, ?, ?, ?)`
-        message.from, message.to, rawEmail, message_id
+        ).bind(
-    ).run();
+            message.from, message.to, rawEmail, message_id
-    if (!success) {
+        ).run();
-        message.setReject(`Failed save message to ${message.to}`);
+        if (!success) {
-        console.log(`Failed save message from ${message.from} to ${message.to}`);
+            message.setReject(`Failed save message to ${message.to}`);
+            console.log(`Failed save message from ${message.from} to ${message.to}`);
+        }
+    }
+    catch (error) {
+        console.log("save email error", error);
     }
 
     // forward email
@@ -55,17 +63,16 @@ async function email(message: ForwardableEmailMessage, env: Bindings, ctx: Execu
     try {
         await sendMailToTelegram(
             { env: env } as Context<HonoCustomType>,
-            message.to, rawEmail, message_id);
+            message.to, parsedEmailContext, message_id);
     } catch (error) {
         console.log("send mail to telegram error", error);
     }
 
     // send webhook
-    let parsedText;
     try {
-        parsedText = await triggerWebhook(
+        await triggerWebhook(
             { env: env } as Context<HonoCustomType>,
-            message.to, rawEmail, message_id
+            message.to, parsedEmailContext, message_id
         );
     } catch (error) {
         console.log("send webhook error", error);
@@ -74,12 +81,10 @@ async function email(message: ForwardableEmailMessage, env: Bindings, ctx: Execu
     // trigger another worker
     try {
         const headersMap = new Map<string, string>();
-        if(message.headers) {
+        if (message.headers) {
-            message.headers.forEach((value, key) => {headersMap.set(key, value);});
+            message.headers.forEach((value, key) => { headersMap.set(key, value); });
-        }
-        if (!parsedText){
-          parsedText = (await commonParseMail(rawEmail))?.text ?? ""
         }
+        const parsedText = (await commonParseMail(parsedEmailContext))?.text ?? ""
         const rpcEmail: RPCEmailMessage = {
             from: message.from,
             to: message.to,

worker/src/mails_api/webhook_settings.ts

@@ -1,5 +1,5 @@
 import { Context } from "hono";
-import { HonoCustomType } from "../types";
+import { HonoCustomType, ParsedEmailContext } from "../types";
 import { CONSTANTS } from "../constants";
 import { AdminWebhookSettings, WebhookSettings } from "../models";
 import { getBooleanValue } from "../utils";
@@ -39,8 +39,8 @@ async function testWebhookSettings(c: Context<HonoCustomType>): Promise<Response
     const { id: mailId, raw } = await c.env.DB.prepare(
         `SELECT id, raw FROM raw_mails WHERE address = ? ORDER BY RANDOM() LIMIT 1`
     ).bind(address).first<{ id: string, raw: string }>() || {};
-
+    const parsedEmailContext: ParsedEmailContext = { rawEmail: raw || "" };
-    const parsedEmail = await commonParseMail(raw);
+    const parsedEmail = await commonParseMail(parsedEmailContext);
     const res = await sendWebhook(settings, {
         id: mailId || "0",
         url: c.env.FRONTEND_URL ? `${c.env.FRONTEND_URL}?mail_id=${mailId}` : "",

worker/src/telegram_api/telegram.ts

@@ -5,7 +5,7 @@ import { callbackQuery } from "telegraf/filters";
 
 import { CONSTANTS } from "../constants";
 import { getDomains, getJsonObjectValue, getStringValue } from '../utils';
-import { HonoCustomType } from "../types";
+import { HonoCustomType, ParsedEmailContext } from "../types";
 import { TelegramSettings } from "./settings";
 import { bindTelegramAddress, deleteTelegramAddress, jwtListToAddressData, tgUserNewAddress, unbindTelegramAddress, unbindTelegramByAddress } from "./common";
 import { commonParseMail } from "../common";
@@ -295,14 +295,14 @@ export async function initTelegramBotCommands(bot: Telegraf) {
 }
 
 const parseMail = async (
-    raw_mail: string | undefined | null,
+    parsedEmailContext: ParsedEmailContext,
     address: string, created_at: string | undefined | null
 ) => {
-    if (!raw_mail) {
+    if (!parsedEmailContext.rawEmail) {
         return {};
     }
     try {
-        const parsedEmail = await commonParseMail(raw_mail);
+        const parsedEmail = await commonParseMail(parsedEmailContext);
         let parsedText = parsedEmail?.text || "";
         if (parsedText.length && parsedText.length > 1000) {
             parsedText = parsedEmail?.text.substring(0, 1000) + "\n\n...\n消息过长请到miniapp查看";
@@ -326,13 +326,14 @@ const parseMail = async (
 
 export async function sendMailToTelegram(
     c: Context<HonoCustomType>, address: string,
-    raw_mail: string, message_id: string | null
+    parsedEmailContext: ParsedEmailContext,
+    message_id: string | null
 ) {
     if (!c.env.TELEGRAM_BOT_TOKEN || !c.env.KV) {
         return;
     }
     const userId = await c.env.KV.get(`${CONSTANTS.TG_KV_PREFIX}:${address}`);
-    const { mail } = await parseMail(raw_mail, address, new Date().toUTCString());
+    const { mail } = await parseMail(parsedEmailContext, address, new Date().toUTCString());
     if (!mail) {
         return;
     }

worker/src/types.d.ts

@@ -44,6 +44,7 @@ export type Bindings = {
     FORWARD_ADDRESS_LIST: string | string[] | undefined
 
     ENABLE_CHECK_JUNK_MAIL: string | boolean | undefined
+    JUNK_MAIL_CHECK_LIST: string | string[] | undefined
     JUNK_MAIL_FORCE_PASS_LIST: string | string[] | undefined
 
     ENABLE_ANOTHER_WORKER: string | boolean | undefined
@@ -108,4 +109,15 @@ type RPCEmailMessage = {
     to: string | undefined | null,
     rawEmail: string | undefined | null,
     headers: Map<string, string>,
-}
+}
+
+type ParsedEmailContext = {
+    rawEmail: string,
+    parsedEmail?: {
+        sender: string,
+        subject: string,
+        text: string,
+        html: string,
+        headers?: Record<string, string>[]
+    } | undefined
+}

worker/wrangler.toml.template

@@ -76,6 +76,8 @@ ENABLE_AUTO_REPLY = false
 # FRONTEND_URL = "https://xxxx.xxx"
 # Enable check junk mail
 # ENABLE_CHECK_JUNK_MAIL = false
+# junk mail check list, if status exists and status is not pass, will be marked as junk mail
+# JUNK_MAIL_CHECK_LIST = = ["spf", "dkim", "dmarc"]
 # junk mail force check pass list, if no status or status is not pass, will be marked as junk mail
 # JUNK_MAIL_FORCE_PASS_LIST = ["spf", "dkim", "dmarc"]
 # Calling other woker to process email
@@ -114,4 +116,4 @@ database_id = "xxx"
 # binding another worker service (parse the code or link), e.g. auth-inbox
 # [[services]]
 # binding = "AUTH_INBOX"
-# service = "auth-inbox"
+# service = "auth-inbox"