GithubBackup/cloudflare_temp_email
1
0
Fork 0
mirror of https://github.com/dreamhunter2333/cloudflare_temp_email.git synced 2026-05-11 18:10:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add JUNK_MAIL_CHECK_LIST for check exits and passed item && add ParsedEmailContext to cache the parsed Email (#553)

Browse Source 
* feat: Junk mail only check JUNK_MAIL_FORCE_PASS_LIST

* feat: add `JUNK_MAIL_CHECK_LIST` for check exits and passed item && add `ParsedEmailContext` to cache the parsed Email
This commit is contained in:
Dream Hunter
2025-01-11 17:42:20 +08:00
committed by GitHub
parent ee3884914b
commit 52caf811f5
12 changed files with 100 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
worker/src/admin_api/mail_webhook_settings.ts
View File

@@ -1,5 +1,5 @@
import { Context } from "hono";
import { HonoCustomType } from "../types";
import { HonoCustomType, ParsedEmailContext } from "../types";
import { CONSTANTS } from "../constants";
import { WebhookSettings } from "../models";
import { commonParseMail, sendWebhook } from "../common";
@@ -25,8 +25,8 @@ async function testWebhookSettings(c: Context<HonoCustomType>): Promise<Response
const { id: mailId, raw } = await c.env.DB.prepare(
`SELECT id, raw FROM raw_mails ORDER BY RANDOM() LIMIT 1`
).first<{ id: string, raw: string }>() || {};
const parsedEmail = await commonParseMail(raw);
const parsedEmailContext: ParsedEmailContext = { rawEmail: raw || "" };
const parsedEmail = await commonParseMail(parsedEmailContext);
const res = await sendWebhook(settings, {
id: mailId || "0",
url: c.env.FRONTEND_URL ? `${c.env.FRONTEND_URL}?mail_id=${mailId}` : "",

1
worker/src/admin_api/worker_config.ts
View File

@@ -43,6 +43,7 @@ export default {
"DISABLE_SHOW_GITHUB": !getBooleanValue(c.env.DISABLE_SHOW_GITHUB),
"DISABLE_ADMIN_PASSWORD_CHECK": getBooleanValue(c.env.DISABLE_ADMIN_PASSWORD_CHECK),
"ENABLE_CHECK_JUNK_MAIL": getBooleanValue(c.env.ENABLE_CHECK_JUNK_MAIL),
"JUNK_MAIL_CHECK_LIST": getStringArray(c.env.JUNK_MAIL_CHECK_LIST),
"JUNK_MAIL_FORCE_PASS_LIST": getStringArray(c.env.JUNK_MAIL_FORCE_PASS_LIST),
"ENABLE_ANOTHER_WORKER": getBooleanValue(c.env.ENABLE_ANOTHER_WORKER),

25
worker/src/common.ts
View File

@@ -2,7 +2,7 @@ import { Context } from 'hono';
import { Jwt } from 'hono/utils/jwt'
import { getBooleanValue, getDomains, getStringValue, getIntValue, getUserRoles, getDefaultDomains, getJsonSetting, getAnotherWorkerList } from './utils';
import { HonoCustomType, UserRole, AnotherWorker, RPCEmailMessage } from './types';
import { HonoCustomType, UserRole, AnotherWorker, RPCEmailMessage, ParsedEmailContext } from './types';
import { unbindTelegramByAddress } from './telegram_api/common';
import { CONSTANTS } from './constants';
import { AdminWebhookSettings, WebhookMail, WebhookSettings } from './models';
@@ -256,16 +256,22 @@ export const handleListQuery = async (
}
export const commonParseMail = async (raw_mail: string | undefined | null): Promise<{
export const commonParseMail = async (parsedEmailContext: ParsedEmailContext): Promise<{
sender: string,
subject: string,
text: string,
html: string,
headers?: Record<string, string>[]
} | undefined> => {
if (!raw_mail) {
// check parsed email context is valid
if (!parsedEmailContext || !parsedEmailContext.rawEmail) {
return undefined;
}
// return parsed email if already parsed
if (parsedEmailContext.parsedEmail) {
return parsedEmailContext.parsedEmail;
}
const raw_mail = parsedEmailContext.rawEmail;
// TODO: WASM parse email
// try {
// const { parse_message_wrapper } = await import('mail-parser-wasm-worker');
@@ -275,7 +281,9 @@ export const commonParseMail = async (raw_mail: string | undefined | null): Prom
// sender: parsedEmail.sender || "",
// subject: parsedEmail.subject || "",
// text: parsedEmail.text || "",
// headers: parsedEmail.headers || [],
// headers: parsedEmail.headers?.map(
// (header) => ({ key: header.key, value: header.value })
// ) || [],
// html: parsedEmail.body_html || "",
// };
// } catch (e) {
@@ -358,9 +366,9 @@ export async function sendWebhook(settings: WebhookSettings, formatMap: WebhookM
export async function triggerWebhook(
c: Context<HonoCustomType>,
address: string,
raw_mail: string,
parsedEmailContext: ParsedEmailContext,
message_id: string | null
): Promise<string | undefined | null> {
): Promise<void> {
if (!c.env.KV || !getBooleanValue(c.env.ENABLE_WEBHOOK)) {
return
}
@@ -391,14 +399,14 @@ export async function triggerWebhook(
`SELECT id FROM raw_mails where address = ? and message_id = ?`
).bind(address, message_id).first<string>("id");
const parsedEmail = await commonParseMail(raw_mail);
const parsedEmail = await commonParseMail(parsedEmailContext);
const webhookMail = {
id: mailId || "",
url: c.env.FRONTEND_URL ? `${c.env.FRONTEND_URL}?mail_id=${mailId}` : "",
from: parsedEmail?.sender || "",
to: address,
subject: parsedEmail?.subject || "",
raw: raw_mail,
raw: parsedEmailContext.rawEmail || "",
parsedText: parsedEmail?.text || "",
parsedHtml: parsedEmail?.html || ""
}
@@ -408,7 +416,6 @@ export async function triggerWebhook(
console.error(res.message);
}
}
return webhookMail.parsedText
}
export async function triggerAnotherWorker(

34
worker/src/email/check_junk.ts
View File

@@ -1,19 +1,22 @@
import { Bindings } from "../types";
import { Bindings, ParsedEmailContext } from "../types";
import { getBooleanValue, getStringArray } from "../utils";
import { commonParseMail } from "../common";
export const check_if_junk_mail = async (
env: Bindings, address: string,
raw_mail: string, message_id: string | null
parsedEmailContext: ParsedEmailContext,
message_id: string | null
): Promise<boolean> => {
if (!getBooleanValue(env.ENABLE_CHECK_JUNK_MAIL)) {
return false;
}
const parsedEmail = await commonParseMail(raw_mail);
const parsedEmail = await commonParseMail(parsedEmailContext);
if (!parsedEmail?.headers) return false;
const checkListWhenExist = getStringArray(env.JUNK_MAIL_CHECK_LIST);
const forcePassList = getStringArray(env.JUNK_MAIL_FORCE_PASS_LIST);
const passedList: string[] = [];
const existList: string[] = [];
const headers = parsedEmail.headers;
for (const header of headers) {
@@ -22,28 +25,35 @@ export const check_if_junk_mail = async (
// check spf
if (header["key"].toLowerCase() == "received-spf") {
if (!header["value"].toLowerCase().includes("pass")) {
return true;
existList.push("spf");
if (header["value"].toLowerCase().includes("pass")) {
passedList.push("spf");
}
passedList.push("spf");
}
// check dkim and dmarc
if (header["key"].toLowerCase() == "authentication-results") {
if (header["value"].toLowerCase().includes("dkim=")) {
if (!header["value"].toLowerCase().includes("dkim=pass")) {
return true;
existList.push("dkim");
if (header["value"].toLowerCase().includes("dkim=pass")) {
passedList.push("dkim");
}
passedList.push("dkim");
}
if (header["value"].toLowerCase().includes("dmarc=")) {
if (!header["value"].toLowerCase().includes("dmarc=pass")) {
return true;
existList.push("dmarc");
if (header["value"].toLowerCase().includes("dmarc=pass")) {
passedList.push("dmarc");
}
passedList.push("dmarc");
}
}
}
// check if all checkListWhenExist item passed when exist
if (checkListWhenExist?.some(
(checkName) => existList.includes(checkName.toLowerCase())
&& !passedList.includes(checkName.toLowerCase())
)) {
return true;
}
if (forcePassList?.length == 0) return false;

45
worker/src/email/index.ts
View File

@@ -2,10 +2,10 @@ import { Context } from "hono";
import { getEnvStringList } from "../utils";
import { sendMailToTelegram } from "../telegram_api";
import { Bindings, HonoCustomType, RPCEmailMessage } from "../types";
import { Bindings, HonoCustomType, RPCEmailMessage, ParsedEmailContext } from "../types";
import { auto_reply } from "./auto_reply";
import { isBlocked } from "./black_list";
import { triggerWebhook, triggerAnotherWorker, commonParseMail} from "../common";
import { triggerWebhook, triggerAnotherWorker, commonParseMail } from "../common";
import { check_if_junk_mail } from "./check_junk";
@@ -16,10 +16,13 @@ async function email(message: ForwardableEmailMessage, env: Bindings, ctx: Execu
return;
}
const rawEmail = await new Response(message.raw).text();
const parsedEmailContext: ParsedEmailContext = {
rawEmail: rawEmail
};
// check if junk mail
try {
const is_junk = await check_if_junk_mail(env, message.to, rawEmail, message.headers.get("Message-ID"));
const is_junk = await check_if_junk_mail(env, message.to, parsedEmailContext, message.headers.get("Message-ID"));
if (is_junk) {
message.setReject("Junk mail");
console.log(`Junk mail from ${message.from} to ${message.to}`);
@@ -31,14 +34,19 @@ async function email(message: ForwardableEmailMessage, env: Bindings, ctx: Execu
const message_id = message.headers.get("Message-ID");
// save email
const { success } = await env.DB.prepare(
`INSERT INTO raw_mails (source, address, raw, message_id) VALUES (?, ?, ?, ?)`
).bind(
message.from, message.to, rawEmail, message_id
).run();
if (!success) {
message.setReject(`Failed save message to ${message.to}`);
console.log(`Failed save message from ${message.from} to ${message.to}`);
try {
const { success } = await env.DB.prepare(
`INSERT INTO raw_mails (source, address, raw, message_id) VALUES (?, ?, ?, ?)`
).bind(
message.from, message.to, rawEmail, message_id
).run();
if (!success) {
message.setReject(`Failed save message to ${message.to}`);
console.log(`Failed save message from ${message.from} to ${message.to}`);
}
}
catch (error) {
console.log("save email error", error);
}
// forward email
@@ -55,17 +63,16 @@ async function email(message: ForwardableEmailMessage, env: Bindings, ctx: Execu
try {
await sendMailToTelegram(
{ env: env } as Context<HonoCustomType>,
message.to, rawEmail, message_id);
message.to, parsedEmailContext, message_id);
} catch (error) {
console.log("send mail to telegram error", error);
}
// send webhook
let parsedText;
try {
parsedText = await triggerWebhook(
await triggerWebhook(
{ env: env } as Context<HonoCustomType>,
message.to, rawEmail, message_id
message.to, parsedEmailContext, message_id
);
} catch (error) {
console.log("send webhook error", error);
@@ -74,12 +81,10 @@ async function email(message: ForwardableEmailMessage, env: Bindings, ctx: Execu
// trigger another worker
try {
const headersMap = new Map<string, string>();
if(message.headers) {
message.headers.forEach((value, key) => {headersMap.set(key, value);});
}
if (!parsedText){
parsedText = (await commonParseMail(rawEmail))?.text ?? ""
if (message.headers) {
message.headers.forEach((value, key) => { headersMap.set(key, value); });
}
const parsedText = (await commonParseMail(parsedEmailContext))?.text ?? ""
const rpcEmail: RPCEmailMessage = {
from: message.from,
to: message.to,

6
worker/src/mails_api/webhook_settings.ts
View File

@@ -1,5 +1,5 @@
import { Context } from "hono";
import { HonoCustomType } from "../types";
import { HonoCustomType, ParsedEmailContext } from "../types";
import { CONSTANTS } from "../constants";
import { AdminWebhookSettings, WebhookSettings } from "../models";
import { getBooleanValue } from "../utils";
@@ -39,8 +39,8 @@ async function testWebhookSettings(c: Context<HonoCustomType>): Promise<Response
const { id: mailId, raw } = await c.env.DB.prepare(
`SELECT id, raw FROM raw_mails WHERE address = ? ORDER BY RANDOM() LIMIT 1`
).bind(address).first<{ id: string, raw: string }>() || {};
const parsedEmail = await commonParseMail(raw);
const parsedEmailContext: ParsedEmailContext = { rawEmail: raw || "" };
const parsedEmail = await commonParseMail(parsedEmailContext);
const res = await sendWebhook(settings, {
id: mailId || "0",
url: c.env.FRONTEND_URL ? `${c.env.FRONTEND_URL}?mail_id=${mailId}` : "",

13
worker/src/telegram_api/telegram.ts
View File

@@ -5,7 +5,7 @@ import { callbackQuery } from "telegraf/filters";
import { CONSTANTS } from "../constants";
import { getDomains, getJsonObjectValue, getStringValue } from '../utils';
import { HonoCustomType } from "../types";
import { HonoCustomType, ParsedEmailContext } from "../types";
import { TelegramSettings } from "./settings";
import { bindTelegramAddress, deleteTelegramAddress, jwtListToAddressData, tgUserNewAddress, unbindTelegramAddress, unbindTelegramByAddress } from "./common";
import { commonParseMail } from "../common";
@@ -295,14 +295,14 @@ export async function initTelegramBotCommands(bot: Telegraf) {
}
const parseMail = async (
raw_mail: string | undefined | null,
parsedEmailContext: ParsedEmailContext,
address: string, created_at: string | undefined | null
) => {
if (!raw_mail) {
if (!parsedEmailContext.rawEmail) {
return {};
}
try {
const parsedEmail = await commonParseMail(raw_mail);
const parsedEmail = await commonParseMail(parsedEmailContext);
let parsedText = parsedEmail?.text || "";
if (parsedText.length && parsedText.length > 1000) {
parsedText = parsedEmail?.text.substring(0, 1000) + "\n\n...\n消息过长请到miniapp查看";
@@ -326,13 +326,14 @@ const parseMail = async (
export async function sendMailToTelegram(
c: Context<HonoCustomType>, address: string,
raw_mail: string, message_id: string | null
parsedEmailContext: ParsedEmailContext,
message_id: string | null
) {
if (!c.env.TELEGRAM_BOT_TOKEN || !c.env.KV) {
return;
}
const userId = await c.env.KV.get(`${CONSTANTS.TG_KV_PREFIX}:${address}`);
const { mail } = await parseMail(raw_mail, address, new Date().toUTCString());
const { mail } = await parseMail(parsedEmailContext, address, new Date().toUTCString());
if (!mail) {
return;
}

14
worker/src/types.d.ts vendored
View File

@@ -44,6 +44,7 @@ export type Bindings = {
FORWARD_ADDRESS_LIST: string | string[] | undefined
ENABLE_CHECK_JUNK_MAIL: string | boolean | undefined
JUNK_MAIL_CHECK_LIST: string | string[] | undefined
JUNK_MAIL_FORCE_PASS_LIST: string | string[] | undefined
ENABLE_ANOTHER_WORKER: string | boolean | undefined
@@ -108,4 +109,15 @@ type RPCEmailMessage = {
to: string | undefined | null,
rawEmail: string | undefined | null,
headers: Map<string, string>,
}
}
type ParsedEmailContext = {
rawEmail: string,
parsedEmail?: {
sender: string,
subject: string,
text: string,
html: string,
headers?: Record<string, string>[]
} | undefined
}