feat: add mail-parser-wasm-worker (#301)

worker/src/common.ts

@@ -166,3 +166,43 @@ export const handleListQuery = async (
     ).bind(...params).first("count") : 0;
     return c.json({ results, count });
 }
+
+
+export const commonParseMail = async (raw_mail: string | undefined | null): Promise<{
+    sender: string,
+    subject: string,
+    text: string,
+    html: string
+} | undefined> => {
+    if (!raw_mail) {
+        return undefined;
+    }
+    // TODO: WASM parse email
+    // try {
+    //     const { parse_message_wrapper } = await import('mail-parser-wasm-worker');
+
+    //     const parsedEmail = parse_message_wrapper(raw_mail);
+    //     return {
+    //         sender: parsedEmail.sender || "",
+    //         subject: parsedEmail.subject || "",
+    //         text: parsedEmail.text || "",
+    //         html: parsedEmail.body_html || "",
+    //     };
+    // } catch (e) {
+    //     console.error("Failed use mail-parser-wasm-worker to parse email", e);
+    // }
+    try {
+        const { default: PostalMime } = await import('postal-mime');
+        const parsedEmail = await PostalMime.parse(raw_mail);
+        return {
+            sender: parsedEmail.from ? `${parsedEmail.from.name} <${parsedEmail.from.address}>` : "",
+            subject: parsedEmail.subject || "",
+            text: parsedEmail.text || "",
+            html: parsedEmail.html || "",
+        };
+    }
+    catch (e) {
+        console.error("Failed use PostalMime to parse email", e);
+    }
+    return undefined;
+}

worker/src/constants.ts

@@ -1,5 +1,5 @@
 export const CONSTANTS = {
-    VERSION: 'v0.5.0',
+    VERSION: 'v0.5.1',
 
     // DB settings
     ADDRESS_BLOCK_LIST_KEY: 'address_block_list',

worker/src/mails_api/webhook_settings.ts

@@ -3,7 +3,7 @@ import { HonoCustomType } from "../types";
 import { CONSTANTS } from "../constants";
 import { AdminWebhookSettings, WebhookMail } from "../models";
 import { getBooleanValue } from "../utils";
-import PostalMime from 'postal-mime';
+import { commonParseMail } from "../common";
 
 
 class WebhookSettings {
@@ -15,10 +15,10 @@ class WebhookSettings {
     body: string = JSON.stringify({
         "from": "${from}",
         "to": "${to}",
-        "headers": "${headers}",
         "subject": "${subject}",
         "raw": "${raw}",
         "parsedText": "${parsedText}",
+        "parsedHtml": "${parsedHtml}",
     }, null, 2)
 }
 
@@ -98,14 +98,14 @@ export async function trigerWebhook(
     if (!settings) {
         return;
     }
-    const parsedEmail = await PostalMime.parse(raw_mail);
+    const parsedEmail = await commonParseMail(raw_mail);
     const res = await sendWebhook(settings, {
-        from: parsedEmail.from.address || "",
+        from: parsedEmail?.sender || "",
         to: address,
-        headers: JSON.stringify(parsedEmail.headers),
-        subject: parsedEmail.subject || "",
+        subject: parsedEmail?.subject || "",
         raw: raw_mail,
-        parsedText: parsedEmail.text || parsedEmail.html || ""
+        parsedText: parsedEmail?.text || "",
+        parsedHtml: parsedEmail?.html || ""
     });
     if (!res.success) {
         console.log(res.message);
@@ -119,14 +119,15 @@ async function testWebhookSettings(c: Context<HonoCustomType>): Promise<Response
     const raw = await c.env.DB.prepare(
         `SELECT raw FROM raw_mails WHERE address = ? ORDER BY RANDOM() LIMIT 1`
     ).bind(address).first<string>("raw");
-    const parsedEmail = raw ? await PostalMime.parse(raw) : {} as any;
+
+    const parsedEmail = await commonParseMail(raw);
     const res = await sendWebhook(settings, {
-        from: parsedEmail?.from?.address || "test@test.com",
+        from: parsedEmail?.sender || "test@test.com",
         to: address,
-        headers: JSON.stringify(parsedEmail?.headers || { "X-Test": "test" }),
         subject: parsedEmail?.subject || "test subject",
         raw: raw || "test raw email",
-        parsedText: parsedEmail?.text || parsedEmail?.html || "test parsed text"
+        parsedText: parsedEmail?.text || "test parsed text",
+        parsedHtml: parsedEmail?.html || "test parsed html"
     });
     if (!res.success) {
         return c.text(res.message || "send webhook error", 400);

worker/src/models/index.ts

@@ -9,10 +9,10 @@ export class AdminWebhookSettings {
 export type WebhookMail = {
     from: string;
     to: string;
-    headers: string;
     subject: string;
     raw: string;
     parsedText: string;
+    parsedHtml: string;
 }
 
 export class CleanupSettings {

worker/src/telegram_api/telegram.ts

@@ -1,15 +1,15 @@
 
 import { Context } from "hono";
-import { Jwt } from 'hono/utils/jwt'
 import { Telegraf, Context as TgContext, Markup } from "telegraf";
 import { callbackQuery } from "telegraf/filters";
-import PostalMime from 'postal-mime';
 
 import { CONSTANTS } from "../constants";
 import { getDomains, getStringValue } from '../utils';
 import { HonoCustomType } from "../types";
 import { TelegramSettings } from "./settings";
 import { bindTelegramAddress, deleteTelegramAddress, jwtListToAddressData, tgUserNewAddress, unbindTelegramAddress, unbindTelegramByAddress } from "./common";
+import { commonParseMail } from "../common";
+
 
 const COMMANDS = [
     {
@@ -195,13 +195,13 @@ export function newTelegramBot(c: Context<HonoCustomType>, token: string): Teleg
         if (!db_address_id) {
             return await ctx.reply("无效地址");
         }
-        const { raw, id: mailId } = await c.env.DB.prepare(
+        const { raw, id: mailId, created_at } = await c.env.DB.prepare(
             `SELECT * FROM raw_mails where address = ? `
             + ` order by id desc limit 1 offset ?`
         ).bind(
             queryAddress, mailIndex
-        ).first<{ raw: string, id: string }>() || {};
-        const { mail } = raw ? await parseMail(raw) : { mail: "已经没有邮件了" };
+        ).first<{ raw: string, id: string, created_at: string }>() || {};
+        const { mail } = raw ? await parseMail(raw, queryAddress, created_at) : { mail: "已经没有邮件了" };
         const settings = await c.env.KV.get<TelegramSettings>(CONSTANTS.TG_KV_SETTINGS_KEY, "json");
         const miniAppButtons = []
         if (settings?.miniAppUrl && settings?.miniAppUrl?.length > 0 && mailId) {
@@ -265,22 +265,26 @@ export async function initTelegramBotCommands(bot: Telegraf) {
     await bot.telegram.setMyCommands(COMMANDS);
 }
 
-const parseMail = async (raw_mail: string | undefined | null) => {
+const parseMail = async (
+    raw_mail: string | undefined | null,
+    address: string, created_at: string | undefined | null
+) => {
     if (!raw_mail) {
         return {};
     }
     try {
-        const parsedEmail = await PostalMime.parse(raw_mail);
-        if (parsedEmail?.text?.length && parsedEmail?.text?.length > 1000) {
-            parsedEmail.text = parsedEmail.text.substring(0, 1000) + "...消息过长请到miniapp查看";
+        const parsedEmail = await commonParseMail(raw_mail);
+        let parsedText = parsedEmail?.text || "";
+        if (parsedText.length && parsedText.length > 1000) {
+            parsedText = parsedEmail?.text.substring(0, 1000) + "\n\n...\n消息过长请到miniapp查看";
         }
         return {
             isHtml: false,
-            mail: `From: ${parsedEmail.from ? `${parsedEmail.from.name}[${parsedEmail.from.address}]` : "无发件人"}\n`
-                + `To: ${parsedEmail.to?.map(t => `${t.name}[${t.address}]`).join(" ")}\n`
-                + `Subject: ${parsedEmail.subject}\n`
-                + `Date: ${parsedEmail.date}\n`
-                + `Content:\n${parsedEmail.text || "解析失败，请打开 mini app 查看"}`
+            mail: `From: ${parsedEmail?.sender || "无发件人"}\n`
+                + `To: ${address}\n`
+                + (created_at ? `Date: ${created_at}\n` : "")
+                + `Subject: ${parsedEmail?.subject}\n`
+                + `Content:\n${parsedEmail?.text || "解析失败，请打开 mini app 查看"}`
         };
     } catch (e) {
         return {
@@ -299,7 +303,7 @@ export async function sendMailToTelegram(
         return;
     }
     const userId = await c.env.KV.get(`${CONSTANTS.TG_KV_PREFIX}:${address}`);
-    const { mail } = await parseMail(raw_mail);
+    const { mail } = await parseMail(raw_mail, address, new Date().toUTCString());
     if (!mail) {
         return;
     }

worker/src/worker.ts

@@ -125,8 +125,14 @@ app.route('/', apiV1)
 app.route('/', apiSendMail)
 app.route('/', telegramApi)
 
-app.get('/', async c => c.text("OK"))
-app.get('/health_check', async c => c.text("OK"))
+app.get('/', async c => {
+	if (!c.env.DB) { return c.text("DB is not available", 400); }
+	return c.text("OK");
+})
+app.get('/health_check', async c => {
+	if (!c.env.DB) { return c.text("DB is not available", 400); }
+	return c.text("OK");
+})
 app.all('/*', async c => c.text("Not Found", 404))