feat: use rust mail-parser (#104)

* feat: imp worker v2

* feat: add rust mail-parser

* feat: imp frontend v2

* feat: imp frontend v2

* feat: update doc

* feat: add mailV1Alert

* feat: remove unused
This commit is contained in:
Dream Hunter
2024-04-09 14:58:19 +08:00
committed by GitHub
parent 105d51ff91
commit def400eb09
27 changed files with 856 additions and 533 deletions

158
worker/src/admin_api.js Normal file
View File

@@ -0,0 +1,158 @@
import { Hono } from 'hono'
import { Jwt } from 'hono/utils/jwt'
const api = new Hono()
api.get('/admin/address', async (c) => {
const { limit, offset, query } = c.req.query();
if (!limit || limit < 0 || limit > 100) {
return c.text("Invalid limit", 400)
}
if (!offset || offset < 0) {
return c.text("Invalid offset", 400)
}
if (query) {
const { results } = await c.env.DB.prepare(
`SELECT * FROM address where concat('${c.env.PREFIX}', name) like ? order by id desc limit ? offset ? `
).bind(`%${query}%`, limit, offset).all();
let count = 0;
if (offset == 0) {
const { count: addressCount } = await c.env.DB.prepare(
`SELECT count(*) as count FROM address where concat('${c.env.PREFIX}', name) like ?`
).bind(`%${query}%`).first();
count = addressCount;
}
return c.json({
results: results.map((r) => {
r.name = c.env.PREFIX + r.name;
return r;
}),
count: count
})
}
const { results } = await c.env.DB.prepare(
`SELECT * FROM address order by id desc limit ? offset ? `
).bind(limit, offset).all();
let count = 0;
if (offset == 0) {
const { count: addressCount } = await c.env.DB.prepare(
`SELECT count(*) as count FROM address`
).first();
count = addressCount;
}
return c.json({
results: results.map((r) => {
r.name = c.env.PREFIX + r.name;
return r;
}),
count: count
})
})
api.delete('/admin/delete_address/:id', async (c) => {
const { id } = c.req.param();
const { success } = await c.env.DB.prepare(
`DELETE FROM address WHERE id = ? `
).bind(id).run();
if (!success) {
return c.text("Failed to delete address", 500)
}
const { success: mailSuccess } = await c.env.DB.prepare(
`DELETE FROM mails WHERE address IN
(select concat('${c.env.PREFIX}', name) from address where id = ?) `
).bind(id).run();
if (!mailSuccess) {
return c.text("Failed to delete mails", 500)
}
return c.json({
success: success
})
})
api.get('/admin/show_password/:id', async (c) => {
const { id } = c.req.param();
const name = await c.env.DB.prepare(
`SELECT name FROM address WHERE id = ? `
).bind(id).first("name");
// compute address
const emailAddress = c.env.PREFIX + name
const jwt = await Jwt.sign({
address: emailAddress,
address_id: id
}, c.env.JWT_SECRET)
return c.json({
password: jwt
})
})
api.get('/admin/mails', async (c) => {
const { address, limit, offset } = c.req.query();
if (!limit || limit < 0 || limit > 100) {
return c.text("Invalid limit", 400)
}
if (!offset || offset < 0) {
return c.text("Invalid offset", 400)
}
const { results } = await c.env.DB.prepare(
`SELECT id, source, raw, created_at FROM raw_mails where address = ? order by id desc limit ? offset ?`
).bind(address, limit, offset).all();
let count = 0;
if (offset == 0) {
const { count: mailCount } = await c.env.DB.prepare(
`SELECT count(*) as count FROM raw_mails where address = ? `
).bind(address).first();
count = mailCount;
}
return c.json({
results: results,
count: count
})
});
api.get('/admin/mails_unknow', async (c) => {
const { limit, offset } = c.req.query();
if (!limit || limit < 0 || limit > 100) {
return c.text("Invalid limit", 400)
}
if (!offset || offset < 0) {
return c.text("Invalid offset", 400)
}
const { results } = await c.env.DB.prepare(`
SELECT id, source, raw, created_at FROM raw_mails
where address NOT IN(select concat('${c.env.PREFIX}', name) from address)
order by id desc limit ? offset ? `
).bind(limit, offset).all();
let count = 0;
if (offset == 0) {
const { count: mailCount } = await c.env.DB.prepare(`
SELECT count(*) as count FROM raw_mails
where address NOT IN
(select concat('${c.env.PREFIX}', name) from address)`
).first();
count = mailCount;
}
return c.json({
results: results,
count: count
})
});
api.get('/admin/statistics', async (c) => {
const { count: mailCount } = await c.env.DB.prepare(`
SELECT count(*) as count FROM mails`
).first();
const { count: addressCount } = await c.env.DB.prepare(`
SELECT count(*) as count FROM address`
).first();
const { count: activeUserCount7days } = await c.env.DB.prepare(`
SELECT count(*) as count FROM address where updated_at > datetime('now', '-7 day')`
).first();
return c.json({
mailCount: mailCount,
userCount: addressCount,
activeUserCount7days: activeUserCount7days
})
});
export { api }

114
worker/src/api_v1.js Normal file
View File

@@ -0,0 +1,114 @@
import { Hono } from 'hono'
// api v1 is deprecated
const api = new Hono()
api.get('/admin/v1/mails', async (c) => {
const { address, limit, offset } = c.req.query();
if (!limit || limit < 0 || limit > 100) {
return c.text("Invalid limit", 400)
}
if (!offset || offset < 0) {
return c.text("Invalid offset", 400)
}
const { results } = await c.env.DB.prepare(
`SELECT id, source, subject, message FROM mails where address = ? order by id desc limit ? offset ? `
).bind(address, limit, offset).all();
let count = 0;
if (offset == 0) {
const { count: mailCount } = await c.env.DB.prepare(
`SELECT count(*) as count FROM mails where address = ? `
).bind(address).first();
count = mailCount;
}
return c.json({
results: results,
count: count
})
});
api.get('/admin/v1/mails_unknow', async (c) => {
const { limit, offset } = c.req.query();
if (!limit || limit < 0 || limit > 100) {
return c.text("Invalid limit", 400)
}
if (!offset || offset < 0) {
return c.text("Invalid offset", 400)
}
const { results } = await c.env.DB.prepare(`
SELECT id, source, subject, message FROM mails
where address NOT IN(select concat('${c.env.PREFIX}', name) from address)
order by id desc limit ? offset ? `
).bind(limit, offset).all();
let count = 0;
if (offset == 0) {
const { count: mailCount } = await c.env.DB.prepare(`
SELECT count(*) as count FROM mails
where address NOT IN
(select concat('${c.env.PREFIX}', name) from address)`
).first();
count = mailCount;
}
return c.json({
results: results,
count: count
})
});
api.get('/api/v1/mails', async (c) => {
const { address } = c.get("jwtPayload")
if (!address) {
return c.json({ "error": "No address" }, 400)
}
const { limit, offset } = c.req.query();
if (!limit || limit < 0 || limit > 100) {
return c.text("Invalid limit", 400)
}
if (!offset || offset < 0) {
return c.text("Invalid offset", 400)
}
const { results } = await c.env.DB.prepare(
`SELECT id, source, subject, message, message_id, created_at FROM mails where address = ? order by id desc limit ? offset ?`
).bind(address, limit, offset).all();
let count = 0;
if (offset == 0) {
const { count: mailCount } = await c.env.DB.prepare(
`SELECT count(*) as count FROM mails where address = ?`
).bind(address).first();
count = mailCount;
}
// add attachments
let attachmentResults = [];
const message_ids = results.map((r) => r.message_id).filter((r) => r);
if (message_ids && message_ids.length > 0) {
const { results: innerAttachmentResults } = await c.env.DB.prepare(
`SELECT id, message_id FROM attachments where message_id in (${message_ids.map((id) => `'${id}'`).join(",")})`
).all();
attachmentResults = innerAttachmentResults || [];
}
results.forEach((r) => {
const attachment_id = attachmentResults.filter((ar) => ar.message_id == r.message_id).map((ar) => ar.id);
if (attachment_id && attachment_id.length > 0) {
r.attachment_id = attachment_id[0];
}
delete r.message_id;
})
return c.json({
results: results,
count: count
})
})
// attachments
api.get("/api/v1/attachment/:attachment_id", async (c) => {
const { attachment_id } = c.req.param();
const { data } = await c.env.DB.prepare(
`SELECT data FROM attachments where id = ? `
).bind(attachment_id).first();
if (!data) {
return c.text("Not found", 404)
}
return c.json(JSON.parse(data))
})
export { api }

View File

@@ -1,8 +1,5 @@
import { createMimeMessage } from "mimetext";
import { EmailMessage } from "cloudflare:email";
import { simpleParser } from 'mailparser';
import PostalMime from 'postal-mime';
global.setImmediate = (callback) => callback();
async function email(message, env, ctx) {
if (env.BLACK_LIST && env.BLACK_LIST.split(",").some(word => message.from.includes(word))) {
@@ -23,34 +20,17 @@ async function email(message, env, ctx) {
}
const message_id = message.headers.get("Message-ID");
let parsedEmail = {};
// todo fix this
if (!message.from.endsWith("@mega.nz")) {
try {
parsedEmail = await simpleParser(rawEmail)
} catch (error) {
console.log(error)
}
}
if (!parsedEmail.html && !parsedEmail.textAsHtml && !parsedEmail.text) {
console.log("Failed parse email, try postal-mime");
parsedEmail = await PostalMime.parse(rawEmail);
}
// process email
// save email
const { success } = await env.DB.prepare(
`INSERT INTO mails (source, address, subject, message, message_id) VALUES (?, ?, ?, ?, ?)`
`INSERT INTO raw_mails (source, address, raw, message_id) VALUES (?, ?, ?, ?)`
).bind(
message.from, message.to,
parsedEmail.subject || "",
parsedEmail.html || parsedEmail.textAsHtml || parsedEmail.text || "",
message_id
message.from, message.to, rawEmail, message_id
).run();
if (!success) {
message.setReject(`Failed save message to ${message.to}`);
console.log(`Failed save message from ${message.from} to ${message.to}`);
}
// auto reply email
try {
const results = await env.DB.prepare(
@@ -80,28 +60,6 @@ async function email(message, env, ctx) {
} catch (error) {
console.log("reply email error", error);
}
// process attachments
try {
if (
env.ENABLE_ATTACHMENT
&& parsedEmail.attachments
&& parsedEmail.attachments.length > 0
) {
const { success } = await env.DB.prepare(
`INSERT INTO attachments (source, address, message_id, data) VALUES (?, ?, ?, ?)`
).bind(
message.from, message.to, message_id,
JSON.stringify(parsedEmail.attachments)
).run();
if (!success) {
message.setReject(`Failed save attachment to ${message.to}`);
console.log(`Failed save attachment from ${message.from} to ${message.to}`);
}
}
}
catch (error) {
console.log("save attachment error", error);
}
} else {
message.setReject(`Unknown address ${message.to}`);
console.log(`Unknown address ${message.to}`);

View File

@@ -16,31 +16,15 @@ api.get('/api/mails', async (c) => {
return c.text("Invalid offset", 400)
}
const { results } = await c.env.DB.prepare(
`SELECT id, source, subject, message, message_id, created_at FROM mails where address = ? order by id desc limit ? offset ?`
`SELECT id, source, raw, created_at FROM raw_mails where address = ? order by id desc limit ? offset ?`
).bind(address, limit, offset).all();
let count = 0;
if (offset == 0) {
const { count: mailCount } = await c.env.DB.prepare(
`SELECT count(*) as count FROM mails where address = ?`
`SELECT count(*) as count FROM raw_mails where address = ?`
).bind(address).first();
count = mailCount;
}
// add attachments
let attachmentResults = [];
const message_ids = results.map((r) => r.message_id).filter((r) => r);
if (message_ids && message_ids.length > 0) {
const { results: innerAttachmentResults } = await c.env.DB.prepare(
`SELECT id, message_id FROM attachments where message_id in (${message_ids.map((id) => `'${id}'`).join(",")})`
).all();
attachmentResults = innerAttachmentResults || [];
}
results.forEach((r) => {
const attachment_id = attachmentResults.filter((ar) => ar.message_id == r.message_id).map((ar) => ar.id);
if (attachment_id && attachment_id.length > 0) {
r.attachment_id = attachment_id[0];
}
delete r.message_id;
})
return c.json({
results: results,
count: count
@@ -84,28 +68,29 @@ api.get('/api/settings', async (c) => {
console.warn("Failed to update address")
}
}
let auto_reply = {};
const results = await c.env.DB.prepare(
`SELECT * FROM auto_reply_mails where address = ? `
).bind(address).first();
if (!results) {
return c.json({
auto_reply: {},
address: address
});
}
return c.json({
auto_reply: {
if (results) {
auto_reply = {
subject: results.subject,
message: results.message,
enabled: results.enabled == 1,
source_prefix: results.source_prefix,
name: results.name,
},
address: address
}
}
const { count: mailCountV1 } = await c.env.DB.prepare(
`SELECT count(*) as count FROM mails where address = ?`
).bind(address).first();
return c.json({
auto_reply: auto_reply,
address: address,
has_v1_mails: mailCountV1 > 0
});
})
api.post('/api/settings', async (c) => {
const { address } = c.get("jwtPayload")
const { auto_reply } = await c.req.json();
@@ -211,169 +196,4 @@ api.delete('/api/delete_address', async (c) => {
})
})
api.get('/admin/address', async (c) => {
const { limit, offset, query } = c.req.query();
if (!limit || limit < 0 || limit > 100) {
return c.text("Invalid limit", 400)
}
if (!offset || offset < 0) {
return c.text("Invalid offset", 400)
}
if (query) {
const { results } = await c.env.DB.prepare(
`SELECT * FROM address where concat('${c.env.PREFIX}', name) like ? order by id desc limit ? offset ? `
).bind(`%${query}%`, limit, offset).all();
let count = 0;
if (offset == 0) {
const { count: addressCount } = await c.env.DB.prepare(
`SELECT count(*) as count FROM address where concat('${c.env.PREFIX}', name) like ?`
).bind(`%${query}%`).first();
count = addressCount;
}
return c.json({
results: results.map((r) => {
r.name = c.env.PREFIX + r.name;
return r;
}),
count: count
})
}
const { results } = await c.env.DB.prepare(
`SELECT * FROM address order by id desc limit ? offset ? `
).bind(limit, offset).all();
let count = 0;
if (offset == 0) {
const { count: addressCount } = await c.env.DB.prepare(
`SELECT count(*) as count FROM address`
).first();
count = addressCount;
}
return c.json({
results: results.map((r) => {
r.name = c.env.PREFIX + r.name;
return r;
}),
count: count
})
})
api.delete('/admin/delete_address/:id', async (c) => {
const { id } = c.req.param();
const { success } = await c.env.DB.prepare(
`DELETE FROM address WHERE id = ? `
).bind(id).run();
if (!success) {
return c.text("Failed to delete address", 500)
}
const { success: mailSuccess } = await c.env.DB.prepare(
`DELETE FROM mails WHERE address IN
(select concat('${c.env.PREFIX}', name) from address where id = ?) `
).bind(id).run();
if (!mailSuccess) {
return c.text("Failed to delete mails", 500)
}
return c.json({
success: success
})
})
api.get('/admin/show_password/:id', async (c) => {
const { id } = c.req.param();
const name = await c.env.DB.prepare(
`SELECT name FROM address WHERE id = ? `
).bind(id).first("name");
// compute address
const emailAddress = c.env.PREFIX + name
const jwt = await Jwt.sign({
address: emailAddress,
address_id: id
}, c.env.JWT_SECRET)
return c.json({
password: jwt
})
})
api.get('/admin/mails', async (c) => {
const { address, limit, offset } = c.req.query();
if (!limit || limit < 0 || limit > 100) {
return c.text("Invalid limit", 400)
}
if (!offset || offset < 0) {
return c.text("Invalid offset", 400)
}
const { results } = await c.env.DB.prepare(
`SELECT id, source, subject, message FROM mails where address = ? order by id desc limit ? offset ? `
).bind(address, limit, offset).all();
let count = 0;
if (offset == 0) {
const { count: mailCount } = await c.env.DB.prepare(
`SELECT count(*) as count FROM mails where address = ? `
).bind(address).first();
count = mailCount;
}
return c.json({
results: results,
count: count
})
});
api.get('/admin/mails_unknow', async (c) => {
const { limit, offset } = c.req.query();
if (!limit || limit < 0 || limit > 100) {
return c.text("Invalid limit", 400)
}
if (!offset || offset < 0) {
return c.text("Invalid offset", 400)
}
const { results } = await c.env.DB.prepare(`
SELECT id, source, subject, message FROM mails
where address NOT IN(select concat('${c.env.PREFIX}', name) from address)
order by id desc limit ? offset ? `
).bind(limit, offset).all();
let count = 0;
if (offset == 0) {
const { count: mailCount } = await c.env.DB.prepare(`
SELECT count(*) as count FROM mails
where address NOT IN
(select concat('${c.env.PREFIX}', name) from address)`
).first();
count = mailCount;
}
return c.json({
results: results,
count: count
})
});
api.get('/admin/statistics', async (c) => {
const { count: mailCount } = await c.env.DB.prepare(`
SELECT count(*) as count FROM mails`
).first();
const { count: addressCount } = await c.env.DB.prepare(`
SELECT count(*) as count FROM address`
).first();
const { count: activeUserCount7days } = await c.env.DB.prepare(`
SELECT count(*) as count FROM address where updated_at > datetime('now', '-7 day')`
).first();
return c.json({
mailCount: mailCount,
userCount: addressCount,
activeUserCount7days: activeUserCount7days
})
});
// attachments
api.get("/api/attachment/:attachment_id", async (c) => {
const { attachment_id } = c.req.param();
const { data } = await c.env.DB.prepare(
`SELECT data FROM attachments where id = ? `
).bind(attachment_id).first();
if (!data) {
return c.text("Not found", 404)
}
return c.json(JSON.parse(data))
})
export { api }

View File

@@ -3,6 +3,8 @@ import { cors } from 'hono/cors';
import { jwt } from 'hono/jwt'
import { api } from './router';
import { api as adminApi } from './admin_api';
import { api as apiV1 } from './api_v1';
import { email } from './email';
const app = new Hono()
@@ -36,8 +38,10 @@ app.use('/admin/*', async (c, next) => {
app.route('/', api)
app.route('/', adminApi)
app.route('/', apiV1)
app.all('/*', async c => c.html(`<h1>Hello World</h1>`))
app.all('/*', async c => c.text("Not Found", 404))
export default {