feat: add s3 attachment (#291)

CHANGELOG.md

@@ -6,6 +6,7 @@
 - UI: 增加本地缓存进行地址管理
 - worker: 增加 `FORWARD_ADDRESS_LIST` 全局邮件转发地址(等同于 `catch all`)
 - UI: 多语言使用路由进行切换
+- 添加保存附件到 S3 的功能
 
 ## v0.4.6
 

frontend/src/api/index.js

@@ -71,6 +71,7 @@ const getOpenSettings = async (message) => {
             copyright: res["copyright"] || openSettings.value.copyright,
             cfTurnstileSiteKey: res["cfTurnstileSiteKey"] || "",
             enableWebhook: res["enableWebhook"] || false,
+            isS3Enabled: res["isS3Enabled"] || false,
         });
         if (openSettings.value.needAuth) {
             showAuth.value = true;

frontend/src/components/MailBox.vue

@@ -35,7 +35,17 @@ const props = defineProps({
     type: Boolean,
     default: false,
     requried: false
-  }
+  },
+  showSaveS3: {
+    type: Boolean,
+    default: false,
+    requried: false
+  },
+  saveToS3: {
+    type: Function,
+    default: (mail_id, filename, blob) => { },
+    requried: false
+  },
 })
 
 const {
@@ -70,7 +80,8 @@ const { t } = useI18n({
       deleteMailTip: 'Are you sure you want to delete this mail?',
       reply: 'Reply',
       showTextMail: 'Show Text Mail',
-      showHtmlMail: 'Show Html Mail'
+      showHtmlMail: 'Show Html Mail',
+      saveToS3: 'Save to S3',
     },
     zh: {
       success: '成功',
@@ -84,7 +95,8 @@ const { t } = useI18n({
       deleteMailTip: '确定要删除这封邮件吗?',
       reply: '回复',
       showTextMail: '显示纯文本邮件',
-      showHtmlMail: '显示HTML邮件'
+      showHtmlMail: '显示HTML邮件',
+      saveToS3: '保存到S3',
     }
   }
 });
@@ -183,6 +195,16 @@ const onSpiltSizeChange = (size) => {
   mailboxSplitSize.value = size;
 }
 
+const attachmentLoding = ref(false)
+const saveToS3Proxy = async (filename, blob) => {
+  attachmentLoding.value = true
+  try {
+    await props.saveToS3(curMail.value.id, filename, blob);
+  } finally {
+    attachmentLoding.value = false
+  }
+}
+
 onMounted(async () => {
   await refresh();
 });
@@ -385,25 +407,31 @@ onBeforeUnmount(() => {
       <template #header>
         <div>{{ t("attachments") }}</div>
       </template>
-      <n-list hoverable clickable>
+      <n-spin v-model:show="attachmentLoding">
-        <n-list-item v-for="row in curAttachments" v-bind:key="row.id">
+        <n-list hoverable clickable>
-          <n-thing class="center" :title="row.filename">
+          <n-list-item v-for="row in curAttachments" v-bind:key="row.id">
-            <template #description>
+            <n-thing class="center" :title="row.filename">
-              <n-space>
+              <template #description>
-                <n-tag type="info">
+                <n-space>
-                  Size: {{ row.size }}
+                  <n-tag type="info">
-                </n-tag>
+                    Size: {{ row.size }}
-              </n-space>
+                  </n-tag>
+                  <n-button v-if="showSaveS3" @click="saveToS3Proxy(row.filename, row.blob)" ghost type="info"
+                    size="small">
+                    {{ t('saveToS3') }}
+                  </n-button>
+                </n-space>
+              </template>
+            </n-thing>
+            <template #suffix>
+              <n-button tag="a" target="_blank" tertiary type="info" size="small" :download="row.filename"
+                :href="row.url">
+                <n-icon :component="CloudDownloadRound" />
+              </n-button>
             </template>
-          </n-thing>
+          </n-list-item>
-          <template #suffix>
+        </n-list>
-            <n-button tag="a" target="_blank" tertiary type="info" size="small" :download="row.filename"
+      </n-spin>
-              :href="row.url">
-              <n-icon :component="CloudDownloadRound" />
-            </n-button>
-          </template>
-        </n-list-item>
-      </n-list>
     </n-modal>
   </div>
 </template>

frontend/src/store/index.js

@@ -18,6 +18,8 @@ export const useGlobalState = createGlobalState(
             domains: [],
             copyright: 'Dream Hunter',
             cfTurnstileSiteKey: '',
+            enableWebhook: false,
+            isS3Enabled: false,
         })
         const settings = ref({
             fetched: false,

frontend/src/utils/email-parser.js

@@ -16,11 +16,11 @@ export async function processItem(item) {
         item.message = parsedEmail.body_html || parsedEmail.text || '';
         item.text = parsedEmail.text || '';
         item.attachments = parsedEmail.attachments?.map((a_item) => {
-            const blob_url = URL.createObjectURL(
+            const blob = new Blob(
-                new Blob(
+                [a_item.content],
-                    [a_item.content],
+                { type: a_item.content_type || 'application/octet-stream' }
-                    { type: a_item.content_type || 'application/octet-stream' }
+            );
-                ))
+            const blob_url = URL.createObjectURL(blob);
             if (a_item.content_id && a_item.content_id.length > 0) {
                 item.message = item.message.replace(`cid:${a_item.content_id}`, blob_url);
             }
@@ -28,7 +28,8 @@ export async function processItem(item) {
                 id: a_item.content_id || Math.random().toString(36).substring(2, 15),
                 filename: a_item.filename || a_item.content_id || "",
                 size: humanFileSize(a_item.content?.length || 0),
-                url: blob_url
+                url: blob_url,
+                blob: blob
             }
         }) || [];
     } catch (error) {
@@ -49,11 +50,11 @@ export async function processItem(item) {
         item.message = parsedEmail.html || parsedEmail.text || item.raw;
         item.text = parsedEmail.text || '';
         item.attachments = parsedEmail.attachments?.map((a_item) => {
-            const blob_url = URL.createObjectURL(
+            const blob = new Blob(
-                new Blob(
+                [a_item.content],
-                    [a_item.content],
+                { type: a_item.mimeType || 'application/octet-stream' }
-                    { type: a_item.mimeType || 'application/octet-stream' }
+            );
-                ))
+            const blob_url = URL.createObjectURL(blob)
             if (a_item.contentId && a_item.contentId.length > 0) {
                 item.message = item.message.replace(`cid:${a_item.contentId}`, blob_url);
             }
@@ -61,7 +62,8 @@ export async function processItem(item) {
                 id: a_item.contentId || Math.random().toString(36).substring(2, 15),
                 filename: a_item.filename || a_item.contentId || "",
                 size: humanFileSize(a_item.content?.length || 0),
-                url: blob_url
+                url: blob_url,
+                blob: blob
             }
         }) || [];
     } catch (error) {

frontend/src/views/Index.vue

@@ -10,11 +10,13 @@ import MailBox from '../components/MailBox.vue';
 import SendBox from '../components/SendBox.vue';
 import AutoReply from './index/AutoReply.vue';
 import AccountSettings from './index/AccountSettings.vue';
-import WenHook from './index/Webhook.vue';
+import Webhook from './index/Webhook.vue';
+import Attachment from './index/Attachment.vue';
 import About from './common/About.vue';
 
 const SendMail = defineAsyncComponent(() => import('./index/SendMail.vue'));
 const { settings, openSettings, indexTab, globalTabplacement } = useGlobalState()
+const message = useMessage()
 
 const { t } = useI18n({
   messages: {
@@ -25,6 +27,8 @@ const { t } = useI18n({
       auto_reply: 'Auto Reply',
       accountSettings: 'Account Settings',
       about: 'About',
+      s3Attachment: 'S3 Attachment',
+      saveToS3Success: 'save to s3 success',
     },
     zh: {
       mailbox: '收件箱',
@@ -33,6 +37,8 @@ const { t } = useI18n({
       auto_reply: '自动回复',
       accountSettings: '账户设置',
       about: '关于',
+      s3Attachment: 'S3附件',
+      saveToS3Success: '保存到s3成功',
     }
   }
 });
@@ -48,6 +54,26 @@ const deleteMail = async (curMailId) => {
 const fetchSenboxData = async (limit, offset) => {
   return await api.fetch(`/api/sendbox?limit=${limit}&offset=${offset}`);
 };
+
+const saveToS3 = async (mail_id, filename, blob) => {
+  try {
+    const { url } = await api.fetch(`/api/attachment/put_url`, {
+      method: 'POST',
+      body: JSON.stringify({ key: `${mail_id}/${filename}` })
+    });
+    // upload to s3 by formdata
+    const formData = new FormData();
+    formData.append(filename, blob);
+    await fetch(url, {
+      method: 'PUT',
+      body: formData
+    });
+    message.success(t('saveToS3Success'));
+  } catch (error) {
+    console.error(error);
+    message.error(error.message || "save to s3 error");
+  }
+}
 </script>
 
 <template>
@@ -55,8 +81,9 @@ const fetchSenboxData = async (limit, offset) => {
     <AddressBar />
     <n-tabs v-if="settings.address" type="card" v-model:value="indexTab" :placement="globalTabplacement">
       <n-tab-pane name="mailbox" :tab="t('mailbox')">
-        <MailBox :showEMailTo="false" :showReply="true" :enableUserDeleteEmail="openSettings.enableUserDeleteEmail"
+        <MailBox :showEMailTo="false" :showReply="true" :showSaveS3="openSettings.isS3Enabled" :saveToS3="saveToS3"
-          :fetchMailData="fetchMailData" :deleteMail="deleteMail" />
+          :enableUserDeleteEmail="openSettings.enableUserDeleteEmail" :fetchMailData="fetchMailData"
+          :deleteMail="deleteMail" />
       </n-tab-pane>
       <n-tab-pane name="sendbox" :tab="t('sendbox')">
         <SendBox :fetchMailData="fetchSenboxData" />
@@ -71,7 +98,10 @@ const fetchSenboxData = async (limit, offset) => {
         <AutoReply />
       </n-tab-pane>
       <n-tab-pane v-if="openSettings.enableWebhook" name="webhook" :tab="t('webhook')">
-        <WenHook />
+        <Webhook />
+      </n-tab-pane>
+      <n-tab-pane v-if="openSettings.isS3Enabled" name="s3_attachment" :tab="t('s3Attachment')">
+        <Attachment />
       </n-tab-pane>
       <n-tab-pane v-if="openSettings.enableIndexAbout" name="about" :tab="t('about')">
         <About />

frontend/src/views/index/Attachment.vue

@@ -0,0 +1,91 @@
+<script setup>
+import { ref, h, onMounted } from 'vue';
+import { useI18n } from 'vue-i18n'
+
+import { api } from '../../api'
+
+const message = useMessage()
+
+const { t } = useI18n({
+    messages: {
+        en: {
+            download: 'Download',
+            action: 'Action',
+        },
+        zh: {
+            download: '下载',
+            action: '操作',
+        }
+    }
+});
+const data = ref([])
+const showDownload = ref(false)
+const curRow = ref({})
+const curDownloadUrl = ref('')
+
+const fetchData = async () => {
+    try {
+        const { results } = await api.fetch(
+            `/api/attachment/list`
+        );
+        data.value = results;
+    } catch (error) {
+        console.log(error)
+        message.error(error.message || "error");
+    }
+}
+
+const columns = [
+    {
+        title: "key",
+        key: "key"
+    },
+    {
+        title: t('action'),
+        key: 'actions',
+        render(row) {
+            return h('div', [
+                h(NButton,
+                    {
+                        type: 'success',
+                        tertiary: true,
+                        onClick: async () => {
+                            try {
+                                const { url } = await api.fetch(`/api/attachment/get_url`, {
+                                    method: 'POST',
+                                    body: JSON.stringify({ key: row.key })
+                                });
+                                curDownloadUrl.value = url;
+                                curRow.value = row;
+                                showDownload.value = true;
+                            }
+                            catch (error) {
+                                console.error(error);
+                                message.error(error.message || "error");
+                            }
+                        }
+                    },
+                    { default: () => t('download') }
+                )
+            ])
+        }
+    }
+]
+
+onMounted(async () => {
+    await fetchData()
+})
+</script>
+
+<template>
+    <div>
+        <n-modal v-model:show="showDownload" preset="dialog" :title="t('download')">
+            <n-tag type="info">{{ curRow.key }}</n-tag>
+            <n-button tag="a" target="_blank" tertiary type="info" size="small" :download="curRow.key.replace('/', '_')"
+                :href="curDownloadUrl">
+                {{ t('download') }}
+            </n-button>
+        </n-modal>
+        <n-data-table :columns="columns" :data="data" :bordered="false" />
+    </div>
+</template>

vitepress-docs/docs/.vitepress/zh.ts

@@ -96,7 +96,7 @@ function sidebarGuide(): DefaultTheme.SidebarItem[] {
         },
         {
             text: '通过命令行部署',
-            collapsed: false,
+            collapsed: true,
             items: [
                 { text: '命令行部署准备', link: 'cli/pre-requisite' },
                 { text: 'D1 数据库', link: 'cli/d1' },
@@ -109,7 +109,7 @@ function sidebarGuide(): DefaultTheme.SidebarItem[] {
         },
         {
             text: '通过用户界面部署',
-            collapsed: false,
+            collapsed: true,
             items: [
                 { text: 'D1 数据库', link: 'ui/d1' },
                 { text: '配置 DKIM', link: 'dkim' },
@@ -121,25 +121,26 @@ function sidebarGuide(): DefaultTheme.SidebarItem[] {
         },
         {
             text: '通过 Github Actions 部署',
-            collapsed: false,
+            collapsed: true,
             items: [
                 { text: '通过 Github Actions 部署', link: 'github-action' },
             ]
         },
         {
             text: '附加功能',
-            collapsed: false,
+            collapsed: true,
             items: [
                 { text: '配置 SMTP IMAP 代理服务', link: 'feature/config-smtp-proxy' },
                 { text: '发送邮件 API', link: 'feature/send-mail-api' },
                 { text: '查看邮件 API', link: 'feature/mail-api' },
                 { text: '配置子域名邮箱', link: 'feature/subdomain' },
                 { text: '配置 Telegram Bot', link: 'feature/telegram' },
+                { text: '配置 S3 附件', link: 'feature/s3-attachment' },
             ]
         },
         {
             text: '功能简介',
-            collapsed: false,
+            collapsed: true,
             items: [
                 { text: 'Admin 控制台', link: 'feature/admin' },
                 { text: 'Admin 用户管理', link: 'feature/admin-user-management' },

vitepress-docs/docs/zh/guide/feature/s3-attachment.md

@@ -0,0 +1,34 @@
+# 配置 S3 附件
+
+## 配置
+
+> [!NOTE]
+> 如果不需要 S3 附件, 可跳过此步骤
+
+在 Cloudflare 创建一个 R2 bucket, 你也可以使用其他的 S3 服务(如有 bug 请提 issue)
+
+参考: [配置 Cloudflare R2 的 cors](https://developers.cloudflare.com/r2/buckets/cors/#add-cors-policies-from-the-dashboard)
+
+参考 [Cloudflare R2  s3 toke](https://developers.cloudflare.com/r2/api/s3/tokens/) 创建 token, 拿到 `ENDPOINT`, `Access Key ID` 和 `Secret Access Key`，然后执行下面的命令添加到 secrets 中
+
+> [!NOTE]
+> 你也可以在 Cloudflare worker 的 UI 界面中添加 `secrets`
+
+```bash
+cd worker
+pnpm wrangler secret put S3_ENDPOINT
+pnpm wrangler secret put S3_ACCESS_KEY_ID
+pnpm wrangler secret put S3_SECRET_ACCESS_KEY
+# 请注意这里的 bucket 是你的 bucket 名称
+pnpm wrangler secret put S3_BUCKET
+```
+
+## 使用
+
+保存附件
+
+![S3 save](/feature/s3-save.png)
+
+下载附件
+
+![S3 download](/public/feature/s3-download.png)

vitepress-docs/docs/zh/guide/feature/telegram.md

@@ -1,5 +1,18 @@
 # 配置 Telegram Bot
 
+## Telegram Bot 配置
+
+> [!NOTE]
+> 如果不需要 Telegram Bot, 可跳过此步骤
+
+请先创建一个 Telegram Bot，然后获取 `token`，然后执行下面的命令，将 `token` 添加到 secrets 中
+
+你也可以在 Cloudflare 的 UI 界面中添加 `secrets`
+
+```bash
+pnpm wrangler secret put TELEGRAM_BOT_TOKEN
+```
+
 ## Bot
 
 - 可设置白名单用户

worker/package.json

@@ -19,6 +19,8 @@
     "wrangler": "^3.57.1"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.588.0",
+    "@aws-sdk/s3-request-presigner": "^3.588.0",
     "hono": "^4.3.9",
     "mimetext": "^3.0.24",
     "postal-mime": "^2.2.5",

worker/src/commom_api.ts

@@ -3,6 +3,7 @@ import { Hono } from 'hono'
 import { getDomains, getPasswords, getBooleanValue } from './utils';
 import { CONSTANTS } from './constants';
 import { HonoCustomType } from './types';
+import { isS3Enabled } from './mails_api/s3_attachment';
 
 const api = new Hono<HonoCustomType>
 
@@ -27,6 +28,7 @@ api.get('/open_api/settings', async (c) => {
         "copyright": c.env.COPYRIGHT,
         "cfTurnstileSiteKey": c.env.CF_TURNSTILE_SITE_KEY,
         "enableWebhook": getBooleanValue(c.env.ENABLE_WEBHOOK),
+        "isS3Enabled": isS3Enabled(c),
         "version": CONSTANTS.VERSION,
     });
 })

worker/src/mails_api/index.ts

@@ -6,6 +6,7 @@ import { newAddress, handleListQuery, deleteAddressWithData } from '../common'
 import { CONSTANTS } from '../constants'
 import auto_reply from './auto_reply'
 import webhook_settings from './webhook_settings';
+import s3_attachment from './s3_attachment';
 
 export const api = new Hono<HonoCustomType>()
 
@@ -14,6 +15,9 @@ api.post('/api/auto_reply', auto_reply.saveAutoReply)
 api.get('/api/webhook/settings', webhook_settings.getWebhookSettings)
 api.post('/api/webhook/settings', webhook_settings.saveWebhookSettings)
 api.post('/api/webhook/test', webhook_settings.testWebhookSettings)
+api.get('/api/attachment/list', s3_attachment.list)
+api.post('/api/attachment/put_url', s3_attachment.getSignedPutUrl)
+api.post('/api/attachment/get_url', s3_attachment.getSignedGetUrl)
 
 api.get('/api/mails', async (c) => {
     const { address } = c.get("jwtPayload")

worker/src/mails_api/s3_attachment.ts

@@ -0,0 +1,84 @@
+import { HonoCustomType } from "../types";
+import { Context } from "hono";
+import {
+    S3Client,
+    ListObjectsV2Command,
+    GetObjectCommand,
+    PutObjectCommand
+} from "@aws-sdk/client-s3";
+import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
+
+export const isS3Enabled = (c: Context<HonoCustomType>) => {
+    return !(!c.env.S3_ENDPOINT ||
+        !c.env.S3_ACCESS_KEY_ID ||
+        !c.env.S3_SECRET_ACCESS_KEY ||
+        !c.env.S3_BUCKET);
+}
+
+const getS3Client = (c: Context<HonoCustomType>) => {
+    if (
+        !c.env.S3_ENDPOINT ||
+        !c.env.S3_ACCESS_KEY_ID ||
+        !c.env.S3_SECRET_ACCESS_KEY ||
+        !c.env.S3_BUCKET
+    ) {
+        throw new Error("S3 config is not set");
+    }
+    return new S3Client({
+        region: "auto",
+        endpoint: c.env.S3_ENDPOINT,
+        credentials: {
+            accessKeyId: c.env.S3_ACCESS_KEY_ID,
+            secretAccessKey: c.env.S3_SECRET_ACCESS_KEY,
+        },
+    });
+}
+
+export default {
+    getSignedGetUrl: async (c: Context<HonoCustomType>) => {
+        const { address } = c.get("jwtPayload")
+        const { key } = await c.req.json()
+        const client = getS3Client(c);
+        const url = await getSignedUrl(
+            client,
+            new GetObjectCommand({
+                Bucket: c.env.S3_BUCKET,
+                Key: `${address}/${key}`
+            }),
+            { expiresIn: c.env.S3_URL_EXPIRES || 360 }
+        );
+        return c.json({ url });
+    },
+    getSignedPutUrl: async (c: Context<HonoCustomType>) => {
+        const { address } = c.get("jwtPayload")
+        const { key } = await c.req.json()
+        const client = getS3Client(c);
+        const url = await getSignedUrl(
+            client,
+            new PutObjectCommand({
+                Bucket: c.env.S3_BUCKET,
+                Key: `${address}/${key}`
+            }),
+            { expiresIn: c.env.S3_URL_EXPIRES || 360 }
+        );
+        return c.json({ url });
+    },
+    list: async (c: Context<HonoCustomType>) => {
+        const { address } = c.get("jwtPayload")
+        const client = getS3Client(c);
+        const data = await client.send(
+            new ListObjectsV2Command({
+                Bucket: c.env.S3_BUCKET,
+                Prefix: `${address}/`
+            })
+        );
+        return c.json(
+            {
+                results: data?.Contents
+                    ?.map((v) => v.Key?.replace(`${address}/`, ""))
+                    ?.filter(k => k)
+                    ?.map((k) => ({ key: k }))
+            }
+        );
+    },
+}

worker/src/types.d.ts

@@ -23,6 +23,13 @@ export type Bindings = {
     COPYRIGHT: string | undefined
     FORWARD_ADDRESS_LIST: string | string[] | undefined
 
+    // s3 config
+    S3_ENDPOINT: string | undefined
+    S3_ACCESS_KEY_ID: string | undefined
+    S3_SECRET_ACCESS_KEY: string | undefined
+    S3_BUCKET: string | undefined
+    S3_URL_EXPIRES: number | undefined
+
     // dkim
     DKIM_SELECTOR: string | undefined
     DKIM_PRIVATE_KEY: string | undefined