mirror of
https://github.com/dreamhunter2333/cloudflare_temp_email.git
synced 2026-09-08 17:08:59 +08:00
fix(imap-proxy): persist IMAP flags and mark mail as read (#1090)
Fix IMAP flag persistence so read/unread state survives reconnects, and align SEARCH/FETCH behavior with persisted flags. Co-authored-by: bounce12340 <bounce12340@users.noreply.github.com>
This commit is contained in:
@@ -142,3 +142,6 @@ pnpm-lock.yaml
|
|||||||
e2e/test-results/
|
e2e/test-results/
|
||||||
e2e/playwright-report/
|
e2e/playwright-report/
|
||||||
e2e/.e2e-pids
|
e2e/.e2e-pids
|
||||||
|
|
||||||
|
# SMTP/IMAP proxy persisted IMAP flags (local dev / bare-metal runs)
|
||||||
|
smtp_proxy_server/data/
|
||||||
|
|||||||
@@ -20,6 +20,9 @@
|
|||||||
- fix: |AI 提取| 强化提示词,要求 AI 保持邮件原始链接域名,避免小模型改写验证链接域名导致错误跳转(issue #1072)
|
- fix: |AI 提取| 强化提示词,要求 AI 保持邮件原始链接域名,避免小模型改写验证链接域名导致错误跳转(issue #1072)
|
||||||
- fix: |AI 提取| HTML-only 邮件在发送给 Workers AI 前会先压缩为可读文本,避免样式模板过长导致验证码位于 4000 字截断之后而无法识别
|
- fix: |AI 提取| HTML-only 邮件在发送给 Workers AI 前会先压缩为可读文本,避免样式模板过长导致验证码位于 4000 字截断之后而无法识别
|
||||||
- fix: |Frontend| 移动端 Header 增加页头内边距,避免标题、菜单按钮与屏幕边缘过近
|
- fix: |Frontend| 移动端 Header 增加页头内边距,避免标题、菜单按钮与屏幕边缘过近
|
||||||
|
- fix: |IMAP 代理| 修复 IMAP `STORE` 无法真正标记邮件已读的问题:邮件不再硬编码为 `\Seen`,且 `SimpleMailbox` 的 flags 变更现持久化到本地 SQLite(新增 `imap_flag_db_path` 配置),使已读/未读状态可在客户端断线重连(如 Thunderbird 轮询)后保留,而非每次新建连接即丢失(issue #1074)
|
||||||
|
- fix: |IMAP 代理| 修复 `SEARCH UNSEEN` 返回全部邮件的问题:`SimpleMailbox.search()` 现按持久化的 flags 计算 `SEEN`/`UNSEEN`/`FLAGGED`/`DELETED`/`ANSWERED`/`DRAFT` 及其否定形式,多个条件按 AND 组合;无法识别的检索条件仍沿用原有行为返回全部邮件
|
||||||
|
- fix: |IMAP 代理| 修复取信不会自动标记已读的问题:`BODY[...]`、`RFC822`、`RFC822.TEXT` 取信现按 RFC 3501 自动置 `\Seen`,而 `BODY.PEEK[...]`、`RFC822.HEADER` 及仅取元数据(如 `FLAGS`)不会
|
||||||
|
|
||||||
### Testing
|
### Testing
|
||||||
|
|
||||||
|
|||||||
@@ -20,6 +20,9 @@
|
|||||||
- fix: |AI Extract| Strengthen the prompt to keep original link domains from the email, preventing small models from rewriting verification-link domains (issue #1072)
|
- fix: |AI Extract| Strengthen the prompt to keep original link domains from the email, preventing small models from rewriting verification-link domains (issue #1072)
|
||||||
- fix: |AI Extract| Convert HTML-only mail bodies into compact readable text before sending them to Workers AI, preventing long templates from pushing verification codes past the 4000-character truncation window
|
- fix: |AI Extract| Convert HTML-only mail bodies into compact readable text before sending them to Workers AI, preventing long templates from pushing verification codes past the 4000-character truncation window
|
||||||
- fix: |Frontend| Add mobile Header page padding so the title and menu button no longer sit too close to the screen edge
|
- fix: |Frontend| Add mobile Header page padding so the title and menu button no longer sit too close to the screen edge
|
||||||
|
- fix: |IMAP Proxy| Fix IMAP `STORE` not actually marking mail as read: messages are no longer hardcoded to `\Seen`, and `SimpleMailbox` flag changes are now persisted to a local SQLite file (new `imap_flag_db_path` setting) so the read/unread state survives a client disconnect and reconnect (e.g. Thunderbird polling) instead of resetting on every new connection (issue #1074)
|
||||||
|
- fix: |IMAP Proxy| Fix `SEARCH UNSEEN` returning every message: `SimpleMailbox.search()` now evaluates `SEEN`/`UNSEEN`/`FLAGGED`/`DELETED`/`ANSWERED`/`DRAFT` and their negations against the persisted flags, combining multiple keys with AND; search keys it cannot evaluate keep the previous behaviour of matching everything
|
||||||
|
- fix: |IMAP Proxy| Fix fetches never marking mail as read: `BODY[...]`, `RFC822` and `RFC822.TEXT` fetches now set `\Seen` per RFC 3501, while `BODY.PEEK[...]`, `RFC822.HEADER` and metadata-only fetches (e.g. `FLAGS`) do not
|
||||||
|
|
||||||
### Testing
|
### Testing
|
||||||
|
|
||||||
|
|||||||
@@ -7,3 +7,4 @@ imap_port=11143
|
|||||||
# imap_tls_key=/path/to/key.pem
|
# imap_tls_key=/path/to/key.pem
|
||||||
# imap_cache_size=500
|
# imap_cache_size=500
|
||||||
# imap_http_timeout=30.0
|
# imap_http_timeout=30.0
|
||||||
|
# imap_flag_db_path=data/imap_flags.db
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ class Settings(BaseSettings):
|
|||||||
imap_tls_key: str = ""
|
imap_tls_key: str = ""
|
||||||
imap_cache_size: int = 500
|
imap_cache_size: int = 500
|
||||||
imap_http_timeout: float = 30.0
|
imap_http_timeout: float = 30.0
|
||||||
|
imap_flag_db_path: str = "data/imap_flags.db"
|
||||||
|
|
||||||
model_config = SettingsConfigDict(env_file=".env")
|
model_config = SettingsConfigDict(env_file=".env")
|
||||||
|
|
||||||
|
|||||||
@@ -12,3 +12,6 @@ services:
|
|||||||
- proxy_url=https://temp-email-api.xxx.xxx
|
- proxy_url=https://temp-email-api.xxx.xxx
|
||||||
- port=8025
|
- port=8025
|
||||||
- imap_port=11143
|
- imap_port=11143
|
||||||
|
volumes:
|
||||||
|
# Persists IMAP flags (e.g. \Seen) across container restarts
|
||||||
|
- ./data:/app/data
|
||||||
|
|||||||
@@ -0,0 +1,178 @@
|
|||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
|
import threading
|
||||||
|
|
||||||
|
|
||||||
|
def _encode_flags(flags: set[str]) -> str:
|
||||||
|
# JSON, not a comma-joined string: commas are legal inside IMAP keywords,
|
||||||
|
# so "foo,bar" as a single keyword must not be split into two on read.
|
||||||
|
return json.dumps(sorted(flags))
|
||||||
|
|
||||||
|
|
||||||
|
def _decode_flags(text: str) -> set[str]:
|
||||||
|
if not text:
|
||||||
|
return set()
|
||||||
|
try:
|
||||||
|
parsed = json.loads(text)
|
||||||
|
if isinstance(parsed, list):
|
||||||
|
return {str(flag) for flag in parsed}
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
# Rows written before the switch to JSON were comma-joined.
|
||||||
|
return set(text.split(","))
|
||||||
|
|
||||||
|
|
||||||
|
class FlagStore:
|
||||||
|
"""Persists IMAP message flags (e.g. \\Seen) to a local SQLite file.
|
||||||
|
|
||||||
|
SimpleMailbox previously kept flags only in an in-memory dict that was
|
||||||
|
recreated from scratch for every IMAP connection (see SimpleRealm.requestAvatar
|
||||||
|
in imap_server.py), so a client's STORE command (e.g. marking a message as
|
||||||
|
read) was silently lost as soon as the session ended. This store gives
|
||||||
|
flags a durable home keyed by (address, mailbox, uid) so they survive
|
||||||
|
reconnects.
|
||||||
|
"""
|
||||||
|
|
||||||
|
_UPSERT_SQL = """
|
||||||
|
INSERT INTO imap_flags (address, mailbox, uid, flags)
|
||||||
|
VALUES (?, ?, ?, ?)
|
||||||
|
ON CONFLICT(address, mailbox, uid) DO UPDATE SET flags = excluded.flags
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, db_path: str):
|
||||||
|
self._db_path = db_path
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
self._init_db()
|
||||||
|
|
||||||
|
def _connect(self) -> sqlite3.Connection:
|
||||||
|
# isolation_level=None: transactions are managed explicitly below, so
|
||||||
|
# sqlite3 must not inject its own implicit BEGIN.
|
||||||
|
return sqlite3.connect(self._db_path, timeout=10, isolation_level=None)
|
||||||
|
|
||||||
|
def _init_db(self):
|
||||||
|
dirname = os.path.dirname(self._db_path)
|
||||||
|
if dirname:
|
||||||
|
os.makedirs(dirname, exist_ok=True)
|
||||||
|
with self._lock:
|
||||||
|
conn = self._connect()
|
||||||
|
try:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
CREATE TABLE IF NOT EXISTS imap_flags (
|
||||||
|
address TEXT NOT NULL,
|
||||||
|
mailbox TEXT NOT NULL,
|
||||||
|
uid INTEGER NOT NULL,
|
||||||
|
flags TEXT NOT NULL,
|
||||||
|
PRIMARY KEY (address, mailbox, uid)
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
def get_all(self, address: str, mailbox: str) -> dict[int, set[str]]:
|
||||||
|
"""Return {uid: flags} for every UID with stored flags in a mailbox."""
|
||||||
|
with self._lock:
|
||||||
|
conn = self._connect()
|
||||||
|
try:
|
||||||
|
rows = conn.execute(
|
||||||
|
"SELECT uid, flags FROM imap_flags WHERE address = ? AND mailbox = ?",
|
||||||
|
(address, mailbox),
|
||||||
|
).fetchall()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
return {uid: _decode_flags(flags) for uid, flags in rows}
|
||||||
|
|
||||||
|
def update_flags(
|
||||||
|
self,
|
||||||
|
address: str,
|
||||||
|
mailbox: str,
|
||||||
|
uids: list[int],
|
||||||
|
flags: set[str],
|
||||||
|
mode: int,
|
||||||
|
) -> dict[int, set[str]]:
|
||||||
|
"""Atomically apply an IMAP STORE to the given UIDs.
|
||||||
|
|
||||||
|
mode follows twisted's convention: 1 adds (+FLAGS), -1 removes
|
||||||
|
(-FLAGS), 0 replaces (FLAGS). Returns {uid: resulting flags}.
|
||||||
|
|
||||||
|
The read-modify-write runs inside one BEGIN IMMEDIATE transaction:
|
||||||
|
two concurrent sessions that loaded the same stale in-memory flags
|
||||||
|
cannot overwrite each other's STORE, because each session's update is
|
||||||
|
recomputed from the row current at its own commit.
|
||||||
|
"""
|
||||||
|
if not uids:
|
||||||
|
return {}
|
||||||
|
flags = set(flags)
|
||||||
|
result: dict[int, set[str]] = {}
|
||||||
|
with self._lock:
|
||||||
|
conn = self._connect()
|
||||||
|
try:
|
||||||
|
conn.execute("BEGIN IMMEDIATE")
|
||||||
|
try:
|
||||||
|
placeholders = ",".join("?" for _ in uids)
|
||||||
|
rows = conn.execute(
|
||||||
|
"SELECT uid, flags FROM imap_flags"
|
||||||
|
" WHERE address = ? AND mailbox = ?"
|
||||||
|
f" AND uid IN ({placeholders})",
|
||||||
|
(address, mailbox, *uids),
|
||||||
|
).fetchall()
|
||||||
|
current = {uid: _decode_flags(text) for uid, text in rows}
|
||||||
|
upserts = []
|
||||||
|
for uid in uids:
|
||||||
|
old = current.get(uid, set())
|
||||||
|
if mode == 1:
|
||||||
|
new = old | flags
|
||||||
|
elif mode == -1:
|
||||||
|
new = old - flags
|
||||||
|
else:
|
||||||
|
new = set(flags)
|
||||||
|
result[uid] = new
|
||||||
|
upserts.append((address, mailbox, uid, _encode_flags(new)))
|
||||||
|
conn.executemany(self._UPSERT_SQL, upserts)
|
||||||
|
conn.execute("COMMIT")
|
||||||
|
except BaseException:
|
||||||
|
conn.execute("ROLLBACK")
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
return result
|
||||||
|
|
||||||
|
def set_flags_bulk(
|
||||||
|
self, address: str, mailbox: str, uid_flags: dict[int, set[str]]
|
||||||
|
) -> None:
|
||||||
|
"""Upsert flags for multiple UIDs in a single transaction."""
|
||||||
|
if not uid_flags:
|
||||||
|
return
|
||||||
|
rows = [
|
||||||
|
(address, mailbox, uid, _encode_flags(flags))
|
||||||
|
for uid, flags in uid_flags.items()
|
||||||
|
]
|
||||||
|
with self._lock:
|
||||||
|
conn = self._connect()
|
||||||
|
try:
|
||||||
|
conn.execute("BEGIN IMMEDIATE")
|
||||||
|
try:
|
||||||
|
conn.executemany(self._UPSERT_SQL, rows)
|
||||||
|
conn.execute("COMMIT")
|
||||||
|
except BaseException:
|
||||||
|
conn.execute("ROLLBACK")
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
_default_store: FlagStore | None = None
|
||||||
|
_default_store_lock = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def get_flag_store() -> FlagStore:
|
||||||
|
"""Return the process-wide FlagStore, created lazily from settings."""
|
||||||
|
global _default_store
|
||||||
|
if _default_store is None:
|
||||||
|
with _default_store_lock:
|
||||||
|
if _default_store is None:
|
||||||
|
from config import settings
|
||||||
|
_default_store = FlagStore(settings.imap_flag_db_path)
|
||||||
|
return _default_store
|
||||||
@@ -3,11 +3,12 @@ import logging
|
|||||||
import time
|
import time
|
||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
|
|
||||||
from twisted.internet import defer
|
from twisted.internet import defer, threads
|
||||||
from twisted.mail import imap4
|
from twisted.mail import imap4
|
||||||
from zope.interface import implementer
|
from zope.interface import implementer
|
||||||
|
|
||||||
from config import settings
|
from config import settings
|
||||||
|
from flag_store import FlagStore, get_flag_store
|
||||||
from imap_http_client import BackendClient
|
from imap_http_client import BackendClient
|
||||||
from imap_message import SimpleMessage
|
from imap_message import SimpleMessage
|
||||||
from parse_email import generate_email_model, parse_email, clean_raw_headers, fix_mojibake
|
from parse_email import generate_email_model, parse_email, clean_raw_headers, fix_mojibake
|
||||||
@@ -51,9 +52,12 @@ class MessageCache:
|
|||||||
@implementer(imap4.IMailboxInfo, imap4.IMailbox, imap4.ISearchableMailbox)
|
@implementer(imap4.IMailboxInfo, imap4.IMailbox, imap4.ISearchableMailbox)
|
||||||
class SimpleMailbox:
|
class SimpleMailbox:
|
||||||
|
|
||||||
def __init__(self, name: str, client: BackendClient):
|
def __init__(self, name: str, client: BackendClient, address: str,
|
||||||
|
flag_store: FlagStore = None):
|
||||||
self.name = name
|
self.name = name
|
||||||
self._client = client
|
self._client = client
|
||||||
|
self._address = address
|
||||||
|
self._flag_store = flag_store if flag_store is not None else get_flag_store()
|
||||||
self.listeners = []
|
self.listeners = []
|
||||||
self.addListener = self.listeners.append
|
self.addListener = self.listeners.append
|
||||||
self.removeListener = self.listeners.remove
|
self.removeListener = self.listeners.remove
|
||||||
@@ -76,7 +80,10 @@ class SimpleMailbox:
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
def getUnseenCount(self):
|
def getUnseenCount(self):
|
||||||
return 0
|
return sum(
|
||||||
|
1 for u in self._uid_index
|
||||||
|
if r"\Seen" not in self._flags.get(u, set())
|
||||||
|
)
|
||||||
|
|
||||||
def isWriteable(self):
|
def isWriteable(self):
|
||||||
return 1
|
return 1
|
||||||
@@ -123,6 +130,7 @@ class SimpleMailbox:
|
|||||||
if count == 0:
|
if count == 0:
|
||||||
self._uid_index = []
|
self._uid_index = []
|
||||||
self._uid_index_built = True
|
self._uid_index_built = True
|
||||||
|
self._flags = {}
|
||||||
return
|
return
|
||||||
|
|
||||||
uid_set = set()
|
uid_set = set()
|
||||||
@@ -146,6 +154,11 @@ class SimpleMailbox:
|
|||||||
|
|
||||||
self._uid_index = sorted(uid_set)
|
self._uid_index = sorted(uid_set)
|
||||||
self._uid_index_built = True
|
self._uid_index_built = True
|
||||||
|
# Load persisted flags (e.g. \Seen set by a previous IMAP session) so
|
||||||
|
# STORE results survive reconnects instead of resetting every session.
|
||||||
|
self._flags = yield threads.deferToThread(
|
||||||
|
self._flag_store.get_all, self._address, self.name
|
||||||
|
)
|
||||||
_logger.info(
|
_logger.info(
|
||||||
"UID index built for %s: %d UIDs, range=%s..%s",
|
"UID index built for %s: %d UIDs, range=%s..%s",
|
||||||
self.name, len(self._uid_index),
|
self.name, len(self._uid_index),
|
||||||
@@ -254,9 +267,10 @@ class SimpleMailbox:
|
|||||||
else:
|
else:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if uid_val not in self._flags:
|
# Flags default to unseen (empty set) unless a previous
|
||||||
self._flags[uid_val] = {r"\Seen"}
|
# STORE persisted them via self._flag_store; self._flags
|
||||||
flags = self._flags[uid_val]
|
# is refreshed from that store in _build_uid_index().
|
||||||
|
flags = self._flags.get(uid_val, set())
|
||||||
msg = SimpleMessage(
|
msg = SimpleMessage(
|
||||||
uid_val, email_model, flags=flags, raw=raw,
|
uid_val, email_model, flags=flags, raw=raw,
|
||||||
created_at=item.get("created_at"),
|
created_at=item.get("created_at"),
|
||||||
@@ -309,18 +323,20 @@ class SimpleMailbox:
|
|||||||
return {}
|
return {}
|
||||||
|
|
||||||
target_uids = self._resolve_message_set(messages, uid)
|
target_uids = self._resolve_message_set(messages, uid)
|
||||||
|
if not target_uids:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
# Apply the delta inside the store's own transaction instead of
|
||||||
|
# computing new flag sets from this session's in-memory copy: another
|
||||||
|
# concurrent session may have STOREd since we loaded, and basing the
|
||||||
|
# write on stale state would silently drop its change.
|
||||||
|
updated_flags = yield threads.deferToThread(
|
||||||
|
self._flag_store.update_flags,
|
||||||
|
self._address, self.name, target_uids, set(flags), mode,
|
||||||
|
)
|
||||||
|
|
||||||
result = {}
|
result = {}
|
||||||
|
for u, current_flags in updated_flags.items():
|
||||||
for u in target_uids:
|
|
||||||
current_flags = self._flags.get(u, set())
|
|
||||||
|
|
||||||
if mode == 1: # +FLAGS
|
|
||||||
current_flags = current_flags | set(flags)
|
|
||||||
elif mode == -1: # -FLAGS
|
|
||||||
current_flags = current_flags - set(flags)
|
|
||||||
elif mode == 0: # FLAGS (replace)
|
|
||||||
current_flags = set(flags)
|
|
||||||
|
|
||||||
self._flags[u] = current_flags
|
self._flags[u] = current_flags
|
||||||
seq = self._uid_to_seq(u)
|
seq = self._uid_to_seq(u)
|
||||||
if seq is not None:
|
if seq is not None:
|
||||||
@@ -328,28 +344,58 @@ class SimpleMailbox:
|
|||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
# SEARCH keys we can answer from persisted flag state, mapped to the flag
|
||||||
|
# and whether it must be present. Anything else falls through to matching
|
||||||
|
# every message, which is how this mailbox answered every query before
|
||||||
|
# flags were persisted.
|
||||||
|
_FLAG_SEARCH_KEYS = {
|
||||||
|
"SEEN": ("\\Seen", True),
|
||||||
|
"UNSEEN": ("\\Seen", False),
|
||||||
|
"DELETED": ("\\Deleted", True),
|
||||||
|
"UNDELETED": ("\\Deleted", False),
|
||||||
|
"FLAGGED": ("\\Flagged", True),
|
||||||
|
"UNFLAGGED": ("\\Flagged", False),
|
||||||
|
"ANSWERED": ("\\Answered", True),
|
||||||
|
"UNANSWERED": ("\\Answered", False),
|
||||||
|
"DRAFT": ("\\Draft", True),
|
||||||
|
"UNDRAFT": ("\\Draft", False),
|
||||||
|
}
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
@defer.inlineCallbacks
|
||||||
def search(self, query, uid):
|
def search(self, query, uid):
|
||||||
if not self._uid_index_built:
|
if not self._uid_index_built:
|
||||||
yield self._build_uid_index()
|
yield self._build_uid_index()
|
||||||
|
|
||||||
results = []
|
predicates = []
|
||||||
|
|
||||||
for term in query:
|
for term in query:
|
||||||
if isinstance(term, str) and term.upper() == "ALL":
|
if isinstance(term, bytes):
|
||||||
if uid:
|
term = term.decode("ascii", "ignore")
|
||||||
results = list(self._uid_index)
|
if not isinstance(term, str):
|
||||||
else:
|
continue
|
||||||
results = list(range(1, len(self._uid_index) + 1))
|
predicate = self._FLAG_SEARCH_KEYS.get(term.upper())
|
||||||
break
|
if predicate is not None:
|
||||||
|
predicates.append(predicate)
|
||||||
|
|
||||||
if not results:
|
matched = [
|
||||||
if uid:
|
u for u in self._uid_index
|
||||||
results = list(self._uid_index)
|
if all(
|
||||||
else:
|
(flag in self._flags.get(u, set())) is present
|
||||||
results = list(range(1, len(self._uid_index) + 1))
|
for flag, present in predicates
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
return results
|
_logger.info(
|
||||||
|
"SEARCH: uid=%s predicates=%d matched=%d/%d",
|
||||||
|
uid, len(predicates), len(matched), len(self._uid_index),
|
||||||
|
)
|
||||||
|
|
||||||
|
if uid:
|
||||||
|
return matched
|
||||||
|
|
||||||
|
return [
|
||||||
|
seq for seq in (self._uid_to_seq(u) for u in matched)
|
||||||
|
if seq is not None
|
||||||
|
]
|
||||||
|
|
||||||
def getUIDNext(self):
|
def getUIDNext(self):
|
||||||
if self._uid_index:
|
if self._uid_index:
|
||||||
|
|||||||
@@ -48,6 +48,49 @@ class SimpleIMAPServer(imap4.IMAP4Server):
|
|||||||
return real_write_seq(data)
|
return real_write_seq(data)
|
||||||
self.transport.writeSequence = logging_write_seq
|
self.transport.writeSequence = logging_write_seq
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _fetch_implies_seen(query):
|
||||||
|
"""Whether a FETCH query marks the messages it returns as read.
|
||||||
|
|
||||||
|
Per RFC 3501, ``BODY[...]`` sets ``\\Seen`` while ``BODY.PEEK[...]``
|
||||||
|
does not. ``RFC822`` and ``RFC822.TEXT`` are defined as equivalents of
|
||||||
|
``BODY[]`` / ``BODY[TEXT]`` and so also set it; ``RFC822.HEADER`` is
|
||||||
|
equivalent to ``BODY.PEEK[HEADER]`` and does not.
|
||||||
|
"""
|
||||||
|
for part in query:
|
||||||
|
part_type = getattr(part, "type", None)
|
||||||
|
if part_type == "body" and not getattr(part, "peek", False):
|
||||||
|
return True
|
||||||
|
if part_type in ("rfc822", "rfc822text"):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def do_FETCH(self, tag, messages, query, uid=0):
|
||||||
|
"""Set ``\\Seen`` for non-peek fetches before delivering the response.
|
||||||
|
|
||||||
|
Twisted hands the parsed query to its own response builder and never
|
||||||
|
passes it to the mailbox, so the mailbox alone cannot tell a ``BODY[]``
|
||||||
|
from a ``BODY.PEEK[]``. Flag the messages here, before the FETCH
|
||||||
|
response is generated, so the flags the client receives are current.
|
||||||
|
"""
|
||||||
|
if not query or not self._fetch_implies_seen(query):
|
||||||
|
return imap4.IMAP4Server.do_FETCH(self, tag, messages, query, uid)
|
||||||
|
|
||||||
|
def _fetch(_):
|
||||||
|
return imap4.IMAP4Server.do_FETCH(self, tag, messages, query, uid)
|
||||||
|
|
||||||
|
def _store_failed(failure):
|
||||||
|
# A failure to persist \Seen must not cost the client its mail.
|
||||||
|
_logger.warning("FETCH: could not set \\Seen: %s", failure.value)
|
||||||
|
return None
|
||||||
|
|
||||||
|
d = defer.maybeDeferred(
|
||||||
|
self.mbox.store, messages, ["\\Seen"], 1, uid
|
||||||
|
)
|
||||||
|
d.addErrback(_store_failed)
|
||||||
|
d.addCallback(_fetch)
|
||||||
|
return d
|
||||||
|
|
||||||
def _cbSelectWork(self, mbox, cmdName, tag):
|
def _cbSelectWork(self, mbox, cmdName, tag):
|
||||||
"""Override to add UIDNEXT in SELECT response (RFC 3501)."""
|
"""Override to add UIDNEXT in SELECT response (RFC 3501)."""
|
||||||
if mbox is None:
|
if mbox is None:
|
||||||
@@ -83,7 +126,7 @@ class Account(imap4.MemoryAccount):
|
|||||||
def _emptyMailbox(self, name, id):
|
def _emptyMailbox(self, name, id):
|
||||||
"""Return a dummy mailbox for CREATE requests (e.g. Gmail creating Drafts)."""
|
"""Return a dummy mailbox for CREATE requests (e.g. Gmail creating Drafts)."""
|
||||||
_logger.debug("Accepting CREATE request for %s", name)
|
_logger.debug("Accepting CREATE request for %s", name)
|
||||||
return SimpleMailbox(name, self._client)
|
return SimpleMailbox(name, self._client, self.name)
|
||||||
|
|
||||||
def create(self, pathspec):
|
def create(self, pathspec):
|
||||||
"""Accept CREATE silently without actually creating mailboxes."""
|
"""Accept CREATE silently without actually creating mailboxes."""
|
||||||
@@ -111,8 +154,8 @@ class SimpleRealm:
|
|||||||
|
|
||||||
client = BackendClient(password)
|
client = BackendClient(password)
|
||||||
|
|
||||||
inbox = SimpleMailbox("INBOX", client)
|
inbox = SimpleMailbox("INBOX", client, username)
|
||||||
sent = SimpleMailbox("SENT", client)
|
sent = SimpleMailbox("SENT", client, username)
|
||||||
|
|
||||||
account = Account(username)
|
account = Account(username)
|
||||||
account._client = client
|
account._client = client
|
||||||
|
|||||||
@@ -0,0 +1,288 @@
|
|||||||
|
"""Regression test for GH issue #1074: IMAP STORE could not mark mail as read.
|
||||||
|
|
||||||
|
Root cause (see 1074-report.md for the full investigation):
|
||||||
|
- smtp_proxy_server/imap_server.py SimpleRealm.requestAvatar() builds a brand
|
||||||
|
new SimpleMailbox (with an empty in-memory `_flags` dict) on every IMAP
|
||||||
|
login/connection.
|
||||||
|
- smtp_proxy_server/imap_mailbox.py SimpleMailbox.store() only ever wrote
|
||||||
|
flag changes into that in-memory dict, and
|
||||||
|
SimpleMailbox._fetch_and_cache_messages() unconditionally defaulted every
|
||||||
|
message's flags to `{\\Seen}`.
|
||||||
|
- Net effect: every message always looked "already read", and a client's
|
||||||
|
STORE +FLAGS (\\Seen) command appeared to succeed but was silently
|
||||||
|
discarded the moment the IMAP session ended (e.g. Thunderbird reconnecting
|
||||||
|
to poll), which matches the reported "cannot mark mail as read".
|
||||||
|
|
||||||
|
This test drives SimpleMailbox directly (no Docker / no live worker
|
||||||
|
backend) against a FakeBackendClient, and verifies that:
|
||||||
|
1. A never-touched message starts unseen (not hardcoded \\Seen).
|
||||||
|
2. STORE +FLAGS (\\Seen) is visible to a brand new SimpleMailbox instance
|
||||||
|
pointed at the same flag_store.FlagStore, simulating a client
|
||||||
|
disconnect + reconnect.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from twisted.internet import defer
|
||||||
|
from twisted.mail import imap4
|
||||||
|
from twisted.python.failure import Failure
|
||||||
|
|
||||||
|
import imap_mailbox
|
||||||
|
from flag_store import FlagStore
|
||||||
|
from imap_mailbox import SimpleMailbox
|
||||||
|
from imap_server import SimpleIMAPServer
|
||||||
|
|
||||||
|
|
||||||
|
def _sync_defer_to_thread(f, *args, **kwargs):
|
||||||
|
"""Stand-in for twisted.internet.threads.deferToThread.
|
||||||
|
|
||||||
|
imap_mailbox.py offloads FlagStore's blocking sqlite calls to a thread
|
||||||
|
via deferToThread. Tests don't run a reactor to service that thread
|
||||||
|
pool, so this executes the callable inline and wraps the result as an
|
||||||
|
already-fired Deferred instead.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
result = f(*args, **kwargs)
|
||||||
|
except Exception:
|
||||||
|
return defer.fail()
|
||||||
|
return defer.succeed(result)
|
||||||
|
|
||||||
|
|
||||||
|
def _run(d):
|
||||||
|
"""Extract the result of a Deferred that fires synchronously."""
|
||||||
|
box = []
|
||||||
|
d.addBoth(box.append)
|
||||||
|
assert box, "Deferred did not fire synchronously"
|
||||||
|
value = box[0]
|
||||||
|
if isinstance(value, Failure):
|
||||||
|
value.raiseException()
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
class FakeBackendClient:
|
||||||
|
"""Minimal stand-in for imap_http_client.BackendClient.
|
||||||
|
|
||||||
|
Serves a fixed in-memory list of message rows so tests don't need a
|
||||||
|
live worker backend or Docker.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, messages):
|
||||||
|
self._messages = messages
|
||||||
|
|
||||||
|
def get_message_count(self, mailbox_name):
|
||||||
|
return defer.succeed(len(self._messages))
|
||||||
|
|
||||||
|
def get_messages(self, mailbox_name, limit, offset):
|
||||||
|
page = self._messages[offset:offset + limit]
|
||||||
|
count = len(self._messages) if offset == 0 else None
|
||||||
|
return defer.succeed((page, count))
|
||||||
|
|
||||||
|
|
||||||
|
class _MailboxTestBase(unittest.TestCase):
|
||||||
|
"""Shared fixture: a temp-dir FlagStore and an inline deferToThread."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self._tmpdir = tempfile.mkdtemp()
|
||||||
|
self._db_path = os.path.join(self._tmpdir, "flags.db")
|
||||||
|
self._orig_deferToThread = imap_mailbox.threads.deferToThread
|
||||||
|
imap_mailbox.threads.deferToThread = _sync_defer_to_thread
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
imap_mailbox.threads.deferToThread = self._orig_deferToThread
|
||||||
|
shutil.rmtree(self._tmpdir, ignore_errors=True)
|
||||||
|
|
||||||
|
def _make_mailbox(self, messages, address="user@example.com"):
|
||||||
|
flag_store = FlagStore(self._db_path)
|
||||||
|
client = FakeBackendClient(messages)
|
||||||
|
return SimpleMailbox("INBOX", client, address, flag_store=flag_store)
|
||||||
|
|
||||||
|
|
||||||
|
class ImapMarkAsSeenPersistenceTest(_MailboxTestBase):
|
||||||
|
|
||||||
|
def test_new_message_starts_unseen(self):
|
||||||
|
mbox = self._make_mailbox([{"id": 1, "raw": ""}])
|
||||||
|
_run(mbox._build_uid_index())
|
||||||
|
self.assertEqual(mbox.getUnseenCount(), 1)
|
||||||
|
|
||||||
|
def test_store_seen_survives_reconnect(self):
|
||||||
|
messages = [{"id": 1, "raw": ""}, {"id": 2, "raw": ""}]
|
||||||
|
|
||||||
|
# --- Session 1: client connects, selects INBOX, sees 2 unseen ---
|
||||||
|
mbox1 = self._make_mailbox(messages)
|
||||||
|
_run(mbox1._build_uid_index())
|
||||||
|
self.assertEqual(mbox1.getUnseenCount(), 2)
|
||||||
|
|
||||||
|
# Client sends: UID STORE 1 +FLAGS (\Seen)
|
||||||
|
message_set = imap4.MessageSet(1, 1)
|
||||||
|
result = _run(mbox1.store(message_set, [r"\Seen"], mode=1, uid=True))
|
||||||
|
self.assertIn(1, [seq for seq in result])
|
||||||
|
self.assertEqual(mbox1.getUnseenCount(), 1)
|
||||||
|
|
||||||
|
# --- Session 2: client disconnects and reconnects (new SimpleMailbox
|
||||||
|
# instance, exactly like SimpleRealm.requestAvatar creates per login) ---
|
||||||
|
mbox2 = self._make_mailbox(messages)
|
||||||
|
_run(mbox2._build_uid_index())
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
mbox2.getUnseenCount(), 1,
|
||||||
|
"flag set via STORE in a previous session was lost on reconnect",
|
||||||
|
)
|
||||||
|
self.assertIn(r"\Seen", mbox2._flags.get(1, set()))
|
||||||
|
self.assertNotIn(r"\Seen", mbox2._flags.get(2, set()))
|
||||||
|
|
||||||
|
def test_store_minus_flags_removes_seen(self):
|
||||||
|
messages = [{"id": 1, "raw": ""}]
|
||||||
|
mbox = self._make_mailbox(messages)
|
||||||
|
_run(mbox._build_uid_index())
|
||||||
|
|
||||||
|
message_set = imap4.MessageSet(1, 1)
|
||||||
|
_run(mbox.store(message_set, [r"\Seen"], mode=1, uid=True))
|
||||||
|
self.assertEqual(mbox.getUnseenCount(), 0)
|
||||||
|
|
||||||
|
_run(mbox.store(message_set, [r"\Seen"], mode=-1, uid=True))
|
||||||
|
self.assertEqual(mbox.getUnseenCount(), 1)
|
||||||
|
|
||||||
|
# The removal must also land in SQLite, not just this instance's
|
||||||
|
# in-memory dict: a fresh mailbox (= reconnect) must see it unseen.
|
||||||
|
mbox2 = self._make_mailbox(messages)
|
||||||
|
_run(mbox2._build_uid_index())
|
||||||
|
self.assertEqual(
|
||||||
|
mbox2.getUnseenCount(), 1,
|
||||||
|
"-FLAGS (\\Seen) was not persisted across reconnect",
|
||||||
|
)
|
||||||
|
self.assertNotIn(r"\Seen", mbox2._flags.get(1, set()))
|
||||||
|
|
||||||
|
def test_keyword_containing_comma_round_trips(self):
|
||||||
|
# Commas are legal in IMAP keywords; "foo,bar" is ONE keyword and
|
||||||
|
# must not come back from storage split into "foo" and "bar".
|
||||||
|
messages = [{"id": 1, "raw": ""}]
|
||||||
|
mbox = self._make_mailbox(messages)
|
||||||
|
_run(mbox._build_uid_index())
|
||||||
|
|
||||||
|
message_set = imap4.MessageSet(1, 1)
|
||||||
|
_run(mbox.store(message_set, ["foo,bar"], mode=1, uid=True))
|
||||||
|
|
||||||
|
mbox2 = self._make_mailbox(messages)
|
||||||
|
_run(mbox2._build_uid_index())
|
||||||
|
self.assertEqual(mbox2._flags.get(1), {"foo,bar"})
|
||||||
|
|
||||||
|
def test_concurrent_sessions_do_not_clobber_each_other(self):
|
||||||
|
# Two sessions load the same (empty) flags, then each STOREs a
|
||||||
|
# different flag on the same message. The second write must not be
|
||||||
|
# computed from its stale in-memory copy, or the first flag is lost.
|
||||||
|
messages = [{"id": 1, "raw": ""}]
|
||||||
|
mbox1 = self._make_mailbox(messages)
|
||||||
|
mbox2 = self._make_mailbox(messages)
|
||||||
|
_run(mbox1._build_uid_index())
|
||||||
|
_run(mbox2._build_uid_index())
|
||||||
|
|
||||||
|
message_set = imap4.MessageSet(1, 1)
|
||||||
|
_run(mbox1.store(message_set, [r"\Seen"], mode=1, uid=True))
|
||||||
|
_run(mbox2.store(message_set, [r"\Flagged"], mode=1, uid=True))
|
||||||
|
|
||||||
|
mbox3 = self._make_mailbox(messages)
|
||||||
|
_run(mbox3._build_uid_index())
|
||||||
|
self.assertEqual(
|
||||||
|
mbox3._flags.get(1), {r"\Seen", r"\Flagged"},
|
||||||
|
"a session's +FLAGS was lost to a concurrent session's stale write",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ImapSearchFlagTest(_MailboxTestBase):
|
||||||
|
"""SEARCH must answer flag keys from persisted state.
|
||||||
|
|
||||||
|
SimpleMailbox declares ISearchableMailbox, so IMAP4Server.do_SEARCH hands
|
||||||
|
the whole query to it and never runs its own search_UNSEEN/search_SEEN
|
||||||
|
helpers. Before this, every SEARCH fell through to "return everything",
|
||||||
|
so `SEARCH UNSEEN` listed mail the user had already read.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def _seen_mailbox(self):
|
||||||
|
"""Two messages, UID 1 marked \\Seen, UID 2 left unread."""
|
||||||
|
messages = [{"id": 1, "raw": ""}, {"id": 2, "raw": ""}]
|
||||||
|
mbox = self._make_mailbox(messages)
|
||||||
|
_run(mbox._build_uid_index())
|
||||||
|
_run(mbox.store(imap4.MessageSet(1, 1), [r"\Seen"], mode=1, uid=True))
|
||||||
|
return mbox
|
||||||
|
|
||||||
|
def test_search_unseen_excludes_seen_messages(self):
|
||||||
|
mbox = self._seen_mailbox()
|
||||||
|
self.assertEqual(_run(mbox.search(["UNSEEN"], uid=True)), [2])
|
||||||
|
|
||||||
|
def test_search_seen_returns_only_seen_messages(self):
|
||||||
|
mbox = self._seen_mailbox()
|
||||||
|
self.assertEqual(_run(mbox.search(["SEEN"], uid=True)), [1])
|
||||||
|
|
||||||
|
def test_search_unseen_returns_sequence_numbers_when_not_uid(self):
|
||||||
|
mbox = self._seen_mailbox()
|
||||||
|
self.assertEqual(_run(mbox.search(["UNSEEN"], uid=False)), [2])
|
||||||
|
|
||||||
|
def test_search_all_still_returns_everything(self):
|
||||||
|
mbox = self._seen_mailbox()
|
||||||
|
self.assertEqual(_run(mbox.search(["ALL"], uid=True)), [1, 2])
|
||||||
|
|
||||||
|
def test_search_accepts_bytes_terms(self):
|
||||||
|
mbox = self._seen_mailbox()
|
||||||
|
self.assertEqual(_run(mbox.search([b"UNSEEN"], uid=True)), [2])
|
||||||
|
|
||||||
|
def test_unsupported_search_key_still_matches_everything(self):
|
||||||
|
# Keys this mailbox cannot evaluate keep the previous lenient
|
||||||
|
# behaviour rather than silently returning an empty result.
|
||||||
|
mbox = self._seen_mailbox()
|
||||||
|
self.assertEqual(_run(mbox.search(["SINCE"], uid=True)), [1, 2])
|
||||||
|
|
||||||
|
def test_combined_keys_are_conjunctive(self):
|
||||||
|
mbox = self._seen_mailbox()
|
||||||
|
_run(mbox.store(imap4.MessageSet(2, 2), [r"\Flagged"], mode=1, uid=True))
|
||||||
|
self.assertEqual(_run(mbox.search(["UNSEEN", "FLAGGED"], uid=True)), [2])
|
||||||
|
self.assertEqual(_run(mbox.search(["SEEN", "FLAGGED"], uid=True)), [])
|
||||||
|
|
||||||
|
|
||||||
|
class FetchImpliesSeenTest(unittest.TestCase):
|
||||||
|
"""Only non-peek body fetches may set \\Seen (RFC 3501 §6.4.5).
|
||||||
|
|
||||||
|
Twisted passes the parsed FETCH query to its own response builder and
|
||||||
|
never to the mailbox, so SimpleMailbox alone cannot distinguish BODY[]
|
||||||
|
from BODY.PEEK[]. SimpleIMAPServer.do_FETCH makes that call instead.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def _implies_seen(self, command):
|
||||||
|
parser = imap4._FetchParser()
|
||||||
|
parser.parseString(command)
|
||||||
|
return SimpleIMAPServer._fetch_implies_seen(parser.result)
|
||||||
|
|
||||||
|
def test_body_sets_seen(self):
|
||||||
|
self.assertTrue(self._implies_seen(b"BODY[]"))
|
||||||
|
|
||||||
|
def test_body_section_sets_seen(self):
|
||||||
|
# BODY[HEADER] is a non-peek fetch and does set \Seen, unlike the
|
||||||
|
# RFC822.HEADER shorthand below.
|
||||||
|
self.assertTrue(self._implies_seen(b"BODY[HEADER]"))
|
||||||
|
|
||||||
|
def test_body_peek_does_not_set_seen(self):
|
||||||
|
self.assertFalse(self._implies_seen(b"BODY.PEEK[]"))
|
||||||
|
|
||||||
|
def test_rfc822_sets_seen(self):
|
||||||
|
self.assertTrue(self._implies_seen(b"RFC822"))
|
||||||
|
|
||||||
|
def test_rfc822_text_sets_seen(self):
|
||||||
|
self.assertTrue(self._implies_seen(b"RFC822.TEXT"))
|
||||||
|
|
||||||
|
def test_rfc822_header_does_not_set_seen(self):
|
||||||
|
# Defined as equivalent to BODY.PEEK[HEADER].
|
||||||
|
self.assertFalse(self._implies_seen(b"RFC822.HEADER"))
|
||||||
|
|
||||||
|
def test_metadata_only_fetch_does_not_set_seen(self):
|
||||||
|
self.assertFalse(self._implies_seen(b"FLAGS"))
|
||||||
|
self.assertFalse(self._implies_seen(b"UID"))
|
||||||
|
self.assertFalse(self._implies_seen(b"RFC822.SIZE"))
|
||||||
|
|
||||||
|
def test_mixed_query_sets_seen_if_any_part_does(self):
|
||||||
|
self.assertTrue(self._implies_seen(b"(FLAGS BODY[])"))
|
||||||
|
self.assertFalse(self._implies_seen(b"(FLAGS BODY.PEEK[])"))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -64,6 +64,7 @@ services:
|
|||||||
| `imap_tls_key` | empty | IMAP TLS private key file path (PEM) |
|
| `imap_tls_key` | empty | IMAP TLS private key file path (PEM) |
|
||||||
| `imap_cache_size` | `500` | Max cached messages per mailbox |
|
| `imap_cache_size` | `500` | Max cached messages per mailbox |
|
||||||
| `imap_http_timeout` | `30.0` | Backend HTTP request timeout (seconds) |
|
| `imap_http_timeout` | `30.0` | Backend HTTP request timeout (seconds) |
|
||||||
|
| `imap_flag_db_path` | `data/imap_flags.db` | SQLite file storing IMAP flags (e.g. `\Seen`) so `STORE` (mark as read) survives client reconnects. Mount a volume over its parent directory so it persists across container restarts |
|
||||||
|
|
||||||
## Enabling STARTTLS
|
## Enabling STARTTLS
|
||||||
|
|
||||||
|
|||||||
@@ -64,6 +64,7 @@ services:
|
|||||||
| `imap_tls_key` | 空 | IMAP TLS 私钥文件路径(PEM) |
|
| `imap_tls_key` | 空 | IMAP TLS 私钥文件路径(PEM) |
|
||||||
| `imap_cache_size` | `500` | 每个邮箱的消息缓存上限 |
|
| `imap_cache_size` | `500` | 每个邮箱的消息缓存上限 |
|
||||||
| `imap_http_timeout` | `30.0` | 后端 HTTP 请求超时时间(秒) |
|
| `imap_http_timeout` | `30.0` | 后端 HTTP 请求超时时间(秒) |
|
||||||
|
| `imap_flag_db_path` | `data/imap_flags.db` | 存储 IMAP 标志(如 `\Seen`)的 SQLite 文件路径,使 `STORE`(标记已读)在客户端断线重连后依然生效。建议挂载卷到其所在目录以在容器重启后保留 |
|
||||||
|
|
||||||
## 启用 STARTTLS
|
## 启用 STARTTLS
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user