From c1c8211c1b5a4d7fa5d0453db2238873fd41a69f Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 11:37:45 +0100 Subject: [PATCH 1/6] fix(outbound): count a missing mailbox only when RCPT TO says so - Issue MAIL, RCPT and DATA one by one so a failure knows which command it answers - Treat 5.1.1, 5.1.2, 5.1.3, 5.1.6 and 5.1.10 as 'no such mailbox' only in answer to RCPT TO - A refused sender (5.1.x answering MAIL FROM) or a refusal after DATA keeps the delivery as failed and never strikes members - Ignore a QUIT error once the relay accepted the message: it caused duplicate deliveries --- README.md | 7 +- internal/outbound/outbound.go | 87 ++++++++++++++++-- internal/outbound/outbound_test.go | 142 ++++++++++++++++++++++++++--- internal/queue/queue_test.go | 30 +++++- 4 files changed, 239 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 8e850ac..a6f52ba 100644 --- a/README.md +++ b/README.md @@ -179,8 +179,11 @@ It answers `451` only when the database itself fails. - A permanent rejection of the message by the relay (5xx: policy, size, line length...) is never retried, but the delivery is not made to vanish: it is kept as **failed**, with the reason, for a week (queue API, `parsimony queue ls`, the Queue page; discard it to clear it). The same happens when a delivery is still failing after 3 days. -- A bad-mailbox reply (`5.1.x`, the address does not exist) drops the delivery and counts as a hard bounce for the - member (3 disable it). Other errors, such as "relay access denied", never disable members. +- A reply to `RCPT TO` that says the address does not exist (`5.1.1`, `5.1.2`, `5.1.3`, `5.1.6`, `5.1.10`) drops the + delivery and counts as a hard bounce for the member (3 disable it). Nothing else counts: not "relay access denied", not a + refused sender (`5.1.7`, `5.1.8`, or any code answering `MAIL FROM`), not a refusal after `DATA`. Those deliveries are kept + as failed. +- A relay that accepted the message is never asked again because `QUIT` went wrong. - Delivery is at-least-once: a crash between the relay accepting a message and the queue entry being removed can send one message to one recipient twice. - `GET /api/v1/queue` and the Queue page show what is pending or failing, and a stuck delivery can be discarded. diff --git a/internal/outbound/outbound.go b/internal/outbound/outbound.go index ffa2603..95a0ef8 100644 --- a/internal/outbound/outbound.go +++ b/internal/outbound/outbound.go @@ -8,6 +8,7 @@ import ( "crypto/x509" "errors" "fmt" + "io" "net" "strings" "sync" @@ -176,10 +177,50 @@ func (r *SMTPRelay) sendBatch(ctx context.Context, from string, to []string, dat } }() - if err := c.SendMail(from, to, bytes.NewReader(data)); err != nil { - return authError(err) + // The commands are issued one by one, rather than with SendMail, to know which one failed. + var mailOpts *smtp.MailOptions + if !isASCII(from) || !allASCII(to) { + mailOpts = &smtp.MailOptions{UTF8: true} } - return c.Quit() + if err := c.Mail(from, mailOpts); err != nil { + return authError(&PhaseError{Phase: "MAIL", Err: err}) + } + for _, rcpt := range to { + if err := c.Rcpt(rcpt, nil); err != nil { + return authError(&PhaseError{Phase: "RCPT", Err: err}) + } + } + w, err := c.Data() + if err != nil { + return authError(&PhaseError{Phase: "DATA", Err: err}) + } + if _, err := io.Copy(w, bytes.NewReader(data)); err != nil { + return &PhaseError{Phase: "DATA", Err: err} + } + if err := w.Close(); err != nil { + return &PhaseError{Phase: "DATA", Err: err} + } + // The relay has accepted the message. If QUIT now fails, retrying would deliver it twice. + c.Quit() + return nil +} + +func isASCII(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] >= 0x80 { + return false + } + } + return true +} + +func allASCII(ss []string) bool { + for _, s := range ss { + if !isASCII(s) { + return false + } + } + return true } // Sent is a message recorded by Fake. @@ -230,15 +271,45 @@ func IsPermanent(err error) bool { return errors.As(err, &se) && se.Code >= 500 && se.Code <= 599 } -// IsBadMailbox reports whether err says the destination mailbox is invalid (SMTP -// enhanced status 5.1.x). Other permanent errors, such as "relay access denied", -// say nothing about the recipient. +// PhaseError says which SMTP command a failure answers. The same enhanced status means different +// things depending on it: 5.1.1 to RCPT is "this mailbox does not exist", but to MAIL FROM it is +// about our own sender address and says nothing about the recipient. +type PhaseError struct { + Phase string // "MAIL", "RCPT" or "DATA" + Err error +} + +func (e *PhaseError) Error() string { return e.Err.Error() } +func (e *PhaseError) Unwrap() error { return e.Err } + +// AtRcpt marks err as the relay's answer to RCPT TO (SMTPRelay does this itself; it is exported +// for tests that fake a relay). +func AtRcpt(err error) error { + if err == nil { + return nil + } + return &PhaseError{Phase: "RCPT", Err: err} +} + +// recipientUnknown are the enhanced status codes that say the destination mailbox is invalid: +// 5.1.1 bad mailbox, 5.1.2 bad domain, 5.1.3 bad address syntax, 5.1.6 moved, 5.1.10 null MX. +var recipientUnknown = map[smtp.EnhancedCode]bool{ + {5, 1, 1}: true, {5, 1, 2}: true, {5, 1, 3}: true, {5, 1, 6}: true, {5, 1, 10}: true, +} + +// IsBadMailbox reports whether err is the relay saying the recipient does not exist: a 5.1.1, 5.1.2, +// 5.1.3, 5.1.6 or 5.1.10 answer to RCPT TO. Anything else, such as "relay access denied", a policy +// refusal or a rejected sender (5.1.7, 5.1.8, or any code answering MAIL FROM), says nothing about +// the recipient and must not count against a member. func IsBadMailbox(err error) bool { var pe *PermanentError if errors.As(err, &pe) { return pe.BadMailbox } + var ph *PhaseError + if !errors.As(err, &ph) || ph.Phase != "RCPT" { + return false + } var se *smtp.SMTPError - return errors.As(err, &se) && se.Code >= 500 && se.Code <= 599 && - se.EnhancedCode[0] == 5 && se.EnhancedCode[1] == 1 + return errors.As(err, &se) && se.Code >= 500 && se.Code <= 599 && recipientUnknown[se.EnhancedCode] } diff --git a/internal/outbound/outbound_test.go b/internal/outbound/outbound_test.go index b845890..a873fee 100644 --- a/internal/outbound/outbound_test.go +++ b/internal/outbound/outbound_test.go @@ -145,13 +145,25 @@ func TestClassifyErrors(t *testing.T) { {"plain", errors.New("connection refused"), false, false}, {"451", smtpErr(451, smtp.EnhancedCode{4, 3, 0}), false, false}, {"421", smtpErr(421, smtp.NoEnhancedCode), false, false}, - {"550 unknown mailbox", smtpErr(550, smtp.EnhancedCode{5, 1, 1}), true, true}, - {"553 bad address syntax", smtpErr(553, smtp.EnhancedCode{5, 1, 3}), true, true}, + {"451 at RCPT", AtRcpt(smtpErr(451, smtp.EnhancedCode{4, 1, 1})), false, false}, {"554 relay denied", smtpErr(554, smtp.EnhancedCode{5, 7, 1}), true, false}, {"550 no enhanced code", smtpErr(550, smtp.NoEnhancedCode), true, false}, - {"wrapped", fmt.Errorf("batch 0-1: %w", smtpErr(550, smtp.EnhancedCode{5, 1, 1})), true, true}, {"explicit permanent", &PermanentError{Err: errors.New("no")}, true, false}, {"explicit bad mailbox", &PermanentError{Err: errors.New("no"), BadMailbox: true}, true, true}, + // The same enhanced code means different things depending on the command it answers. + {"5.1.1 without a command", smtpErr(550, smtp.EnhancedCode{5, 1, 1}), true, false}, + {"5.1.1 at RCPT", AtRcpt(smtpErr(550, smtp.EnhancedCode{5, 1, 1})), true, true}, + {"5.1.2 at RCPT", AtRcpt(smtpErr(550, smtp.EnhancedCode{5, 1, 2})), true, true}, + {"5.1.3 at RCPT", AtRcpt(smtpErr(553, smtp.EnhancedCode{5, 1, 3})), true, true}, + {"5.1.6 at RCPT", AtRcpt(smtpErr(550, smtp.EnhancedCode{5, 1, 6})), true, true}, + {"5.1.10 at RCPT", AtRcpt(smtpErr(550, smtp.EnhancedCode{5, 1, 10})), true, true}, + {"wrapped, at RCPT", AtRcpt(fmt.Errorf("batch 0-1: %w", smtpErr(550, smtp.EnhancedCode{5, 1, 1}))), true, true}, + {"5.1.0 at RCPT", AtRcpt(smtpErr(550, smtp.EnhancedCode{5, 1, 0})), true, false}, + {"5.1.4 at RCPT", AtRcpt(smtpErr(550, smtp.EnhancedCode{5, 1, 4})), true, false}, + {"5.1.7 at RCPT (bad sender)", AtRcpt(smtpErr(553, smtp.EnhancedCode{5, 1, 7})), true, false}, + {"5.1.8 at RCPT (bad sender)", AtRcpt(smtpErr(553, smtp.EnhancedCode{5, 1, 8})), true, false}, + {"5.2.2 at RCPT (mailbox full)", AtRcpt(smtpErr(552, smtp.EnhancedCode{5, 2, 2})), true, false}, + {"5.7.1 at RCPT (policy)", AtRcpt(smtpErr(554, smtp.EnhancedCode{5, 7, 1})), true, false}, } for _, c := range cases { if got := IsPermanent(c.err); got != c.perm { @@ -163,28 +175,40 @@ func TestClassifyErrors(t *testing.T) { } } -type rejecting struct{ err error } +// rejecting refuses one stage of the SMTP transaction. +type rejecting struct{ mail, rcpt, data error } -func (r rejecting) NewSession(*smtp.Conn) (smtp.Session, error) { return &rejectSession{r.err}, nil } +func (r rejecting) NewSession(*smtp.Conn) (smtp.Session, error) { return &rejectSession{r}, nil } -type rejectSession struct{ err error } +type rejectSession struct{ r rejecting } func (s *rejectSession) Reset() {} func (s *rejectSession) Logout() error { return nil } -func (s *rejectSession) Mail(string, *smtp.MailOptions) error { return nil } -func (s *rejectSession) Rcpt(string, *smtp.RcptOptions) error { return s.err } -func (s *rejectSession) Data(io.Reader) error { return nil } +func (s *rejectSession) Mail(string, *smtp.MailOptions) error { return s.r.mail } +func (s *rejectSession) Rcpt(string, *smtp.RcptOptions) error { return s.r.rcpt } +func (s *rejectSession) Data(io.Reader) error { return s.r.data } -func TestRelayRejectionsAreClassifiedFromTheWire(t *testing.T) { +func se(code int, enh smtp.EnhancedCode, msg string) error { + return &smtp.SMTPError{Code: code, EnhancedCode: enh, Message: msg} +} + +func TestRelayRejectionsAreClassifiedByTheCommandTheyAnswer(t *testing.T) { for name, tc := range map[string]struct { - err error + r rejecting perm, mailbox bool }{ - "unknown user": {&smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"}, true, true}, - "relay denied": {&smtp.SMTPError{Code: 554, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "relay access denied"}, true, false}, - "try later": {&smtp.SMTPError{Code: 451, EnhancedCode: smtp.EnhancedCode{4, 3, 0}, Message: "try later"}, false, false}, + "unknown user at RCPT": {rejecting{rcpt: se(550, smtp.EnhancedCode{5, 1, 1}, "no such user")}, true, true}, + "relay denied at RCPT": {rejecting{rcpt: se(554, smtp.EnhancedCode{5, 7, 1}, "relay access denied")}, true, false}, + "try later at RCPT": {rejecting{rcpt: se(451, smtp.EnhancedCode{4, 3, 0}, "try later")}, false, false}, + "bad sender 5.1.8 at MAIL": {rejecting{mail: se(550, smtp.EnhancedCode{5, 1, 8}, "bad outbound sender")}, true, false}, + "sender domain 5.1.0 at MAIL": {rejecting{mail: se(550, smtp.EnhancedCode{5, 1, 0}, "sender domain not found")}, true, false}, + "unlisted sender 5.1.7 at MAIL": {rejecting{mail: se(553, smtp.EnhancedCode{5, 1, 7}, "sender address rejected")}, true, false}, + "even 5.1.1 at MAIL is the sender": {rejecting{mail: se(550, smtp.EnhancedCode{5, 1, 1}, "sender unknown")}, true, false}, + "5.1.1 after DATA is not a mailbox": {rejecting{data: se(550, smtp.EnhancedCode{5, 1, 1}, "content refused")}, true, false}, + "policy refusal after DATA": {rejecting{data: se(554, smtp.EnhancedCode{5, 7, 1}, "spam")}, true, false}, + "temporary after DATA": {rejecting{data: se(451, smtp.EnhancedCode{4, 3, 0}, "try later")}, false, false}, } { - srv := smtp.NewServer(rejecting{tc.err}) + srv := smtp.NewServer(tc.r) ln, _ := net.Listen("tcp", "127.0.0.1:0") go srv.Serve(ln) r := &SMTPRelay{Addr: ln.Addr().String()} @@ -199,6 +223,94 @@ func TestRelayRejectionsAreClassifiedFromTheWire(t *testing.T) { } } +// quitDropper is a bare SMTP server that accepts the message and then hangs up on QUIT. +type quitDropper struct { + mu sync.Mutex + accepted int +} + +func startQuitDropper(t *testing.T, reply string) (*quitDropper, string) { + t.Helper() + q := &quitDropper{} + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go func() { + defer c.Close() + c.Write([]byte("220 test ESMTP\r\n")) + buf := make([]byte, 0, 4096) + one := make([]byte, 1) + inData := false + line := func() (string, bool) { + buf = buf[:0] + for { + if _, err := c.Read(one); err != nil { + return "", false + } + buf = append(buf, one[0]) + if len(buf) >= 2 && buf[len(buf)-2] == '\r' && buf[len(buf)-1] == '\n' { + return string(buf[:len(buf)-2]), true + } + } + } + for { + l, ok := line() + if !ok { + return + } + switch { + case inData: + if l == "." { + inData = false + q.mu.Lock() + q.accepted++ + q.mu.Unlock() + c.Write([]byte("250 2.0.0 queued\r\n")) + } + case strings.HasPrefix(l, "EHLO"), strings.HasPrefix(l, "HELO"): + c.Write([]byte("250 test\r\n")) + case strings.HasPrefix(l, "MAIL"), strings.HasPrefix(l, "RCPT"): + c.Write([]byte("250 2.1.0 ok\r\n")) + case l == "DATA": + inData = true + c.Write([]byte("354 go ahead\r\n")) + case l == "QUIT": + if reply != "" { + c.Write([]byte(reply)) + } + return // hang up + } + } + }() + } + }() + return q, ln.Addr().String() +} + +func TestAFailedQuitAfterTheMessageWasAcceptedIsNotAFailure(t *testing.T) { + // The relay said 250 to the message: retrying because QUIT misbehaved would deliver it twice. + for name, reply := range map[string]string{"hang up without a reply": "", "421 instead of 221": "421 4.4.2 closing early\r\n"} { + q, addr := startQuitDropper(t, reply) + err := (&SMTPRelay{Addr: addr}).Send(context.Background(), "b@x.org", []string{"a@x.com"}, []byte("Subject: x\r\n\r\nx\r\n")) + if err != nil { + t.Errorf("%s: Send must succeed, the message was accepted: %v", name, err) + } + q.mu.Lock() + if q.accepted != 1 { + t.Errorf("%s: accepted %d times", name, q.accepted) + } + q.mu.Unlock() + } +} + // silent accepts connections and never says a word. func silent(t *testing.T) string { t.Helper() diff --git a/internal/queue/queue_test.go b/internal/queue/queue_test.go index 5945d70..7cd9f76 100644 --- a/internal/queue/queue_test.go +++ b/internal/queue/queue_test.go @@ -251,7 +251,7 @@ func TestBadMailboxCountsAHardBounceAndDisablesAfterThree(t *testing.T) { ms, _ := e.s.Members(l.ID) id := ms[0].ID e.snd.fn = func(context.Context, string, []string) error { - return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"} + return outbound.AtRcpt(&smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"}) } for i := 1; i <= 3; i++ { e.s.Enqueue([]byte("Subject: x\r\n\r\nx"), []store.NewOutbound{{From: "b@x.org", Rcpt: "gone@x.com", MemberID: id}}) @@ -293,7 +293,7 @@ func TestOtherPermanentRejectionsNeverCountAgainstTheMember(t *testing.T) { func TestBadMailboxForSystemMailOrUnknownMemberIsHarmless(t *testing.T) { e := setup(t) e.snd.fn = func(context.Context, string, []string) error { - return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"} + return outbound.AtRcpt(&smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"}) } e.s.Enqueue([]byte("x"), []store.NewOutbound{ {From: "b@x.org", Rcpt: "a@x.com", MemberID: 0}, // confirmation mail @@ -532,3 +532,29 @@ func TestEachAttemptIsSignedAgain(t *testing.T) { t.Fatalf("a retry is signed afresh (new timestamp), signing calls = %d", calls) } } + +func TestARefusedSenderIsNotAMissingMailbox(t *testing.T) { + // A relay that rejects the list's own sender address (5.1.8, 5.1.7, 5.1.0 or even 5.1.1 answering + // MAIL FROM) says nothing about the recipients: the members must not be struck, and the mail must + // stay visible as failed instead of vanishing. + e := setup(t) + l, _ := e.s.CreateList("dev", "example.org", "Dev") + e.s.AddMember(l.ID, "ok@x.com", "") + ms, _ := e.s.Members(l.ID) + e.snd.fn = func(context.Context, string, []string) error { + return &outbound.PhaseError{Phase: "MAIL", Err: &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 8}, Message: "Bad outbound sender"}} + } + for i := 0; i < 5; i++ { + e.s.Enqueue([]byte("x"), []store.NewOutbound{{From: "dev+bounce-1-x@example.org", Rcpt: "ok@x.com", MemberID: ms[0].ID}}) + st, _ := e.w.RunOnce(context.Background()) + if st.Failed != 1 || st.Bounced != 0 || st.Dropped != 0 { + t.Fatalf("round %d: %+v", i+1, st) + } + } + if m, _ := e.s.MemberByID(ms[0].ID); m.BounceCount != 0 || m.Disabled { + t.Fatalf("the member was struck for our own sender problem: %+v", m) + } + if stats, _ := e.s.OutboxStats(); stats.Failed != 5 { + t.Fatalf("every delivery stays listed as failed: %+v", stats) + } +} From 94b5a58e932b58ec45ad8560f875598df6354d49 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 11:39:21 +0100 Subject: [PATCH 2/6] fix(store): moderation listings never read message bodies, bound the held bytes - Store subject, From header and size when a message is held; ListHeld returns the summary only - Cap one held message at 10 MiB (ErrHeldTooLarge) and the bytes held per list at 200 MiB (ErrHeldFull) - Summarise the messages already held when migrating an existing database, one at a time - Show the From header and whether it differs from the envelope sender in the held API - Thread listings no longer load the bodies of the thread --- internal/api/api.go | 24 +++--- internal/api/api_test.go | 41 +++++++++-- internal/store/m2_test.go | 141 +++++++++++++++++++++++++++++++++++ internal/store/store.go | 151 ++++++++++++++++++++++++++++++-------- 4 files changed, 310 insertions(+), 47 deletions(-) diff --git a/internal/api/api.go b/internal/api/api.go index c0481ec..b40b9ab 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -2,13 +2,11 @@ package api import ( - "bytes" "context" "crypto/subtle" "encoding/json" "errors" "fmt" - "mime" "net/http" "net/mail" "strconv" @@ -353,18 +351,17 @@ type heldDTO struct { Size int `json:"size"` CreatedAt time.Time `json:"created_at"` Reason string `json:"reason"` // why the message was held + // The From header as received, and whether it names someone other than the envelope sender: + // what recipients will see as the author, against who really sent it. + From string `json:"from"` + Mismatch bool `json:"mismatch"` } -func subjectOf(data []byte) string { - m, err := mail.ReadMessage(bytes.NewReader(data)) - if err != nil { - return "" - } - subj := m.Header.Get("Subject") - if dec, err := new(mime.WordDecoder).DecodeHeader(subj); err == nil { - return dec - } - return subj +// fromMismatch reports whether the From header names an address other than the envelope sender, or +// no usable address at all. +func fromMismatch(fromHeader, sender string) bool { + a, err := mail.ParseAddress(fromHeader) + return err != nil || !strings.EqualFold(a.Address, sender) } func (s *server) listHeld(w http.ResponseWriter, r *http.Request) { @@ -383,7 +380,8 @@ func (s *server) listHeld(w http.ResponseWriter, r *http.Request) { } out := make([]heldDTO, 0, len(hs)) for _, h := range hs { - out = append(out, heldDTO{ID: h.ID, Sender: h.Sender, Subject: subjectOf(h.Data), Size: len(h.Data), CreatedAt: h.CreatedAt, Reason: h.Reason}) + out = append(out, heldDTO{ID: h.ID, Sender: h.Sender, Subject: h.Subject, Size: h.Size, CreatedAt: h.CreatedAt, Reason: h.Reason, + From: h.FromHeader, Mismatch: fromMismatch(h.FromHeader, h.Sender)}) } writeJSON(w, http.StatusOK, out) } diff --git a/internal/api/api_test.go b/internal/api/api_test.go index ff8d4bc..ac1e3ad 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -271,11 +271,13 @@ func heldServer(t *testing.T) (http.Handler, *store.Store, *fakeReleaser) { } type heldJSON struct { - ID int64 `json:"id"` - Sender string `json:"sender"` - Subject string `json:"subject"` - Size int `json:"size"` - Reason string `json:"reason"` + ID int64 `json:"id"` + Sender string `json:"sender"` + From string `json:"from"` + Mismatch bool `json:"mismatch"` + Subject string `json:"subject"` + Size int `json:"size"` + Reason string `json:"reason"` } func TestHeldQueueAPI(t *testing.T) { @@ -298,6 +300,35 @@ func TestHeldQueueAPI(t *testing.T) { expect(t, do(t, h, "DELETE", "/api/v1/held/abc", "", true), 404) } +func TestHeldListingShowsWhoReallyWrote(t *testing.T) { + s, _ := store.Open(":memory:") + defer s.Close() + l, _ := s.CreateList("dev", "example.org", "Dev") + s.HoldMessage(l.ID, "bob@x.com", []byte("From: Bob \r\nSubject: honest\r\n\r\nb"), "") + s.HoldMessage(l.ID, "bob@x.com", []byte("From: Eve \r\nSubject: passing as Eve\r\n\r\nb"), "") + s.HoldMessage(l.ID, "bob@x.com", []byte("Subject: no From at all\r\n\r\nb"), "") + s.HoldMessage(l.ID, "bob@x.com", []byte("From: not an address\r\nSubject: garbled\r\n\r\nb"), "") + h := New(s, token, nil) + + var hs []heldJSON + decode(t, do(t, h, "GET", "/api/v1/lists/1/held", "", true), &hs) + if len(hs) != 4 { + t.Fatalf("%+v", hs) + } + want := []struct { + from string + mismatch bool + }{{"Bob ", false}, {"Eve ", true}, {"", true}, {"not an address", true}} + for i, w := range want { + if hs[i].From != w.from || hs[i].Mismatch != w.mismatch { + t.Errorf("held %d: from %q mismatch %v, want %q %v", i+1, hs[i].From, hs[i].Mismatch, w.from, w.mismatch) + } + if hs[i].Sender != "bob@x.com" { + t.Errorf("the envelope sender stays visible: %q", hs[i].Sender) + } + } +} + func TestHeldRequiresAuth(t *testing.T) { h, _, _ := heldServer(t) expect(t, do(t, h, "POST", "/api/v1/held/1/approve", "", false), 401) diff --git a/internal/store/m2_test.go b/internal/store/m2_test.go index 7ac36c8..d1f03d0 100644 --- a/internal/store/m2_test.go +++ b/internal/store/m2_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "path/filepath" + "strings" "testing" "time" ) @@ -326,3 +327,143 @@ func TestOldHeldMessagesExpire(t *testing.T) { t.Fatalf("expiry is applied to the list being written to, not to others: %v", err) } } + +func TestHeldListingsCarryASummaryAndNeverTheBody(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + body := strings.Repeat("0123456789abcdef\r\n", 1000) + data := []byte("From: =?UTF-8?Q?Jos=C3=A9?= \r\nSubject: =?UTF-8?Q?Caff=C3=A8_a_las_nueve?=\r\n\r\n" + body) + id, err := s.HoldMessage(l.ID, "Alice@Example.com", data, "r") + if err != nil { + t.Fatal(err) + } + list, err := s.ListHeld(l.ID) + if err != nil || len(list) != 1 { + t.Fatalf("%v %v", list, err) + } + h := list[0] + if h.Data != nil { + t.Fatalf("a listing must not read the %d-byte body into memory", len(data)) + } + if h.Subject != "Caffè a las nueve" || h.FromHeader != "=?UTF-8?Q?Jos=C3=A9?= " || h.Size != len(data) || h.Sender != "alice@example.com" { + t.Fatalf("summary: %+v", h) + } + full, err := s.GetHeld(id) + if err != nil || len(full.Data) != len(data) || full.Subject != h.Subject { + t.Fatalf("a single message still comes with its body: %v %d", err, len(full.Data)) + } +} + +func TestHeldSummaryOfOddMessages(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + long := strings.Repeat("x", 500) + for name, tc := range map[string]struct { + data string + subject, from string + }{ + "no headers": {"\r\njust a body\r\n", "", ""}, + "empty": {"", "", ""}, + "no subject": {"From: a@x.com\r\n\r\nb", "", "a@x.com"}, + "long subject": {"Subject: " + long + "\r\n\r\nb", strings.Repeat("x", 200), ""}, + "long from": {"From: " + long + "\r\n\r\nb", "", strings.Repeat("x", 200)}, + "folded subject": {"Subject: one\r\n two\r\n\r\nb", "one two", ""}, + "control characters": {"Subject: a\x01b\x1b[31mred\r\n\r\nb", "ab[31mred", ""}, + } { + id, err := s.HoldMessage(l.ID, "a@x.com", []byte(tc.data), "") + if err != nil { + t.Fatalf("%s: %v", name, err) + } + h, _ := s.GetHeld(id) + if h.Subject != tc.subject || h.FromHeader != tc.from { + t.Errorf("%s: subject %q from %q, want %q %q", name, h.Subject, h.FromHeader, tc.subject, tc.from) + } + } +} + +func TestHeldQueueHasAByteCapAndAMessageCap(t *testing.T) { + oldMsg, oldList := heldMessageLimit, heldBytesLimit + heldMessageLimit, heldBytesLimit = 1000, 3500 + t.Cleanup(func() { heldMessageLimit, heldBytesLimit = oldMsg, oldList }) + s := newTestStore(t) + a, _ := s.CreateList("a", "example.org", "A") + b, _ := s.CreateList("b", "example.org", "B") + msg := func(n int) []byte { return []byte("Subject: s\r\n\r\n" + strings.Repeat("x", n-14)) } + + if _, err := s.HoldMessage(a.ID, "x@y.com", msg(1001), ""); !errors.Is(err, ErrHeldTooLarge) { + t.Fatalf("one message over the limit: want ErrHeldTooLarge, got %v", err) + } + if _, err := s.HoldMessage(a.ID, "x@y.com", msg(1000), ""); err != nil { + t.Fatalf("exactly at the limit is fine: %v", err) + } + var ids []int64 + for i := 0; i < 2; i++ { // 1000 + 1000 + 1000 = 3000 of 3500 + id, err := s.HoldMessage(a.ID, "x@y.com", msg(1000), "") + if err != nil { + t.Fatal(err) + } + ids = append(ids, id) + } + if _, err := s.HoldMessage(a.ID, "x@y.com", msg(600), ""); !errors.Is(err, ErrHeldFull) { + t.Fatalf("3000 + 600 > 3500: want ErrHeldFull, got %v", err) + } + if _, err := s.HoldMessage(a.ID, "x@y.com", msg(500), ""); err != nil { + t.Fatalf("3000 + 500 = 3500 fits: %v", err) + } + if _, err := s.HoldMessage(b.ID, "x@y.com", msg(1000), ""); err != nil { + t.Fatalf("another list has its own allowance: %v", err) + } + if err := s.DeleteHeld(ids[0]); err != nil { + t.Fatal(err) + } + if _, err := s.HoldMessage(a.ID, "x@y.com", msg(900), ""); err != nil { + t.Fatalf("moderating frees room: %v", err) + } +} + +func TestMigrationSummarisesExistingHeldMessages(t *testing.T) { + path := t.TempDir() + "/old.db" + s, _ := Open(path) + l, _ := s.CreateList("dev", "example.org", "Dev") + data := []byte("From: Bob \r\nSubject: Old held\r\n\r\nbody") + id, _ := s.HoldMessage(l.ID, "bob@x.com", data, "r") + s.db.Exec(`ALTER TABLE held DROP COLUMN subject`) + s.db.Exec(`ALTER TABLE held DROP COLUMN from_header`) + s.db.Exec(`ALTER TABLE held DROP COLUMN size`) + s.Close() + + s2, err := Open(path) + if err != nil { + t.Fatalf("migrate: %v", err) + } + defer s2.Close() + list, _ := s2.ListHeld(l.ID) + if len(list) != 1 || list[0].ID != id || list[0].Subject != "Old held" || list[0].FromHeader != "Bob " || list[0].Size != len(data) || list[0].Data != nil { + t.Fatalf("an existing held message is summarised by the migration: %+v", list) + } +} + +func TestThreadListingsDoNotLoadBodies(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + root, _, _ := s.ArchiveMessage(l.ID, msg("", "Root", strings.Repeat("big body ", 1000))) + reply := msg("", "Re: Root", "reply body") + reply.InReplyTo = "" + s.ArchiveMessage(l.ID, reply) + th, err := s.ThreadMessages(l.ID, root) + if err != nil || len(th) != 2 { + t.Fatalf("%v %v", th, err) + } + for _, m := range th { + if m.Body != "" { + t.Fatalf("a thread listing needs subjects, not bodies: %q", m.Body[:20]) + } + if m.Size == 0 || m.Subject == "" { + t.Fatalf("summary: %+v", m) + } + } + one, _ := s.GetMessage(root) + if !strings.HasPrefix(one.Body, "big body") { + t.Fatal("a single message still has its body") + } +} diff --git a/internal/store/store.go b/internal/store/store.go index cb27ebb..5abaedf 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -2,13 +2,17 @@ package store import ( + "bytes" "crypto/rand" "database/sql" "encoding/hex" "errors" "fmt" + "mime" + "net/mail" "strings" "time" + "unicode" _ "modernc.org/sqlite" ) @@ -18,6 +22,8 @@ var ( ErrNotFound = errors.New("not found") // ErrHeldFull is returned when a list's moderation queue has no more room. ErrHeldFull = errors.New("moderation queue is full") + // ErrHeldTooLarge is returned for a message too big to be held for moderation. + ErrHeldTooLarge = errors.New("message too large to hold for moderation") // ErrExists is returned when creating something that already exists. ErrExists = errors.New("already exists") ) @@ -43,6 +49,12 @@ const ( ActionLeave = "leave" ) +// Limits on the moderation queue of one list. They are variables so the tests can lower them. +var ( + heldMessageLimit = 10 << 20 // bytes in one held message + heldBytesLimit = 200 << 20 // bytes held for one list, all messages together +) + // BounceThreshold is the number of hard bounces after which a member is disabled. const BounceThreshold = 3 @@ -94,6 +106,11 @@ type Held struct { Data []byte CreatedAt time.Time Reason string // why the message was held, for the admin + + // What a listing needs, stored when the message is held so that a listing never has to read a body. + Subject string // decoded, at most 200 characters + FromHeader string // the From header as received, at most 200 characters + Size int // bytes of the message } // Pending is an unconfirmed subscribe or unsubscribe request. @@ -137,7 +154,10 @@ CREATE TABLE IF NOT EXISTS held ( sender TEXT NOT NULL, data BLOB NOT NULL, created_at INTEGER NOT NULL, - reason TEXT NOT NULL DEFAULT '' + reason TEXT NOT NULL DEFAULT '', + subject TEXT NOT NULL DEFAULT '', + from_header TEXT NOT NULL DEFAULT '', + size INTEGER NOT NULL DEFAULT 0 ); CREATE TABLE IF NOT EXISTS pending ( token TEXT PRIMARY KEY, @@ -275,11 +295,82 @@ func migrate(db *sql.DB) error { return err } } + if has, _ := hasColumn(db, "held", "size"); !has { + if err := addHeldSummary(db); err != nil { + return err + } + } } _, err := db.Exec(schema) return err } +// addHeldSummary adds the summary columns of the held queue and fills them for the messages +// already there, one message at a time so that a big queue is not loaded into memory. +func addHeldSummary(db *sql.DB) error { + for _, stmt := range []string{ + `ALTER TABLE held ADD COLUMN subject TEXT NOT NULL DEFAULT ''`, + `ALTER TABLE held ADD COLUMN from_header TEXT NOT NULL DEFAULT ''`, + `ALTER TABLE held ADD COLUMN size INTEGER NOT NULL DEFAULT 0`, + `UPDATE held SET size=length(data)`, + } { + if _, err := db.Exec(stmt); err != nil { + return err + } + } + rows, err := db.Query(`SELECT id FROM held`) + if err != nil { + return err + } + var ids []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + rows.Close() + return err + } + ids = append(ids, id) + } + rows.Close() + for _, id := range ids { + var data []byte + if err := db.QueryRow(`SELECT data FROM held WHERE id=?`, id).Scan(&data); err != nil { + return err + } + subject, from := summarize(data) + if _, err := db.Exec(`UPDATE held SET subject=?, from_header=? WHERE id=?`, subject, from, id); err != nil { + return err + } + } + return nil +} + +// summarize extracts what a moderation listing shows: the decoded subject and the From header, both +// without control characters and cut to 200 characters. +func summarize(data []byte) (subject, from string) { + m, err := mail.ReadMessage(bytes.NewReader(data)) + if err != nil { + return "", "" + } + clean := func(s string) string { + s = strings.Map(func(r rune) rune { + if unicode.IsControl(r) { + return -1 + } + return r + }, strings.Join(strings.Fields(s), " ")) + if r := []rune(s); len(r) > 200 { + s = string(r[:200]) + } + return s + } + raw := m.Header.Get("Subject") + if dec, err := new(mime.WordDecoder).DecodeHeader(raw); err == nil { + raw = dec + } + return clean(raw), clean(m.Header.Get("From")) +} + // Close closes the database. func (s *Store) Close() error { return s.db.Close() } @@ -468,34 +559,47 @@ func (s *Store) RecordHardBounce(memberID int64, threshold int) (bool, error) { // Held messages. -// HoldMessage queues a message for moderation and returns its id. reason tells the admin why. +// HoldMessage queues a message for moderation and returns its id. reason tells the admin why. A list +// holds a bounded amount: ErrHeldTooLarge for one message over heldMessageLimit, ErrHeldFull when the +// number of messages or the bytes held for the list would be exceeded. func (s *Store) HoldMessage(listID int64, sender string, data []byte, reason string) (int64, error) { if data == nil { data = []byte{} // the column is NOT NULL } + if len(data) > heldMessageLimit { + return 0, ErrHeldTooLarge + } now := s.Now() if _, err := s.db.Exec(`DELETE FROM held WHERE list_id=? AND created_at= maxHeldPerList { + if n >= maxHeldPerList || bytesHeld+len(data) > heldBytesLimit { return 0, ErrHeldFull } - res, err := s.db.Exec(`INSERT INTO held(list_id, sender, data, created_at, reason) VALUES(?,?,?,?,?)`, - listID, norm(sender), data, now.Unix(), reason) + subject, from := summarize(data) + res, err := s.db.Exec(`INSERT INTO held(list_id, sender, data, created_at, reason, subject, from_header, size) VALUES(?,?,?,?,?,?,?,?)`, + listID, norm(sender), data, now.Unix(), reason, subject, from, len(data)) if err != nil { return 0, err } return res.LastInsertId() } -func scanHeld(row interface{ Scan(...any) error }) (Held, error) { +const heldCols = `id, list_id, sender, created_at, reason, subject, from_header, size` + +func scanHeld(row interface{ Scan(...any) error }, withData bool) (Held, error) { var h Held var ts int64 - err := row.Scan(&h.ID, &h.ListID, &h.Sender, &h.Data, &ts, &h.Reason) + var err error + if withData { + err = row.Scan(&h.ID, &h.ListID, &h.Sender, &ts, &h.Reason, &h.Subject, &h.FromHeader, &h.Size, &h.Data) + } else { + err = row.Scan(&h.ID, &h.ListID, &h.Sender, &ts, &h.Reason, &h.Subject, &h.FromHeader, &h.Size) + } if errors.Is(err, sql.ErrNoRows) { return Held{}, ErrNotFound } @@ -503,16 +607,17 @@ func scanHeld(row interface{ Scan(...any) error }) (Held, error) { return h, err } -// ListHeld returns the held messages of a list, oldest first. +// ListHeld returns the held messages of a list, oldest first. It carries the summary of each message +// and never its body (Data is nil): a listing must not read up to 25 MiB per message into memory. func (s *Store) ListHeld(listID int64) ([]Held, error) { - rows, err := s.db.Query(`SELECT id, list_id, sender, data, created_at, reason FROM held WHERE list_id=? ORDER BY id`, listID) + rows, err := s.db.Query(`SELECT `+heldCols+` FROM held WHERE list_id=? ORDER BY id`, listID) if err != nil { return nil, err } defer rows.Close() var out []Held for rows.Next() { - h, err := scanHeld(rows) + h, err := scanHeld(rows, false) if err != nil { return nil, err } @@ -521,9 +626,9 @@ func (s *Store) ListHeld(listID int64) ([]Held, error) { return out, rows.Err() } -// GetHeld returns a held message by id. +// GetHeld returns one held message with its body. func (s *Store) GetHeld(id int64) (Held, error) { - return scanHeld(s.db.QueryRow(`SELECT id, list_id, sender, data, created_at, reason FROM held WHERE id=?`, id)) + return scanHeld(s.db.QueryRow(`SELECT `+heldCols+`, data FROM held WHERE id=?`, id), true) } // DeleteHeld discards a held message. @@ -885,20 +990,8 @@ func (s *Store) SearchMessages(listID int64, query string, limit, offset int) ([ return out, total, err } -// ThreadMessages returns every message of a thread, oldest first, with bodies. +// ThreadMessages returns every message of a thread, oldest first, without bodies: a thread listing +// shows subjects, and a thread can hold many messages of up to 1 MiB of text each. func (s *Store) ThreadMessages(listID, threadID int64) ([]Message, error) { - rows, err := s.db.Query(`SELECT `+msgCols+`, body FROM archive WHERE list_id=? AND thread_id=? ORDER BY date, id`, listID, threadID) - if err != nil { - return nil, err - } - defer rows.Close() - var out []Message - for rows.Next() { - m, err := scanMessage(rows, true) - if err != nil { - return nil, err - } - out = append(out, m) - } - return out, rows.Err() + return s.queryMessages(`SELECT `+msgCols+` FROM archive WHERE list_id=? AND thread_id=? ORDER BY date, id`, listID, threadID) } From 1e46be79b016480eb5d0182fcc7d8aa8b37c91a7 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 11:39:41 +0100 Subject: [PATCH 3/6] fix(ingest): answer 552 for a message too big to hold, with tests --- internal/ingest/ingest.go | 2 ++ internal/ingest/ingest_test.go | 27 +++++++++++++++++++++++++++ internal/router/identity_test.go | 13 +++++++++++++ internal/router/router.go | 2 ++ 4 files changed, 44 insertions(+) diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 45920a6..02f2beb 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -92,6 +92,8 @@ func (s *session) status(rcpt string, err error) error { return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such list"} case errors.Is(err, router.ErrHeldFull): return &smtp.SMTPError{Code: 452, EnhancedCode: smtp.EnhancedCode{4, 2, 2}, Message: "the moderation queue of this list is full, try again later"} + case errors.Is(err, router.ErrHeldTooLarge): + return &smtp.SMTPError{Code: 552, EnhancedCode: smtp.EnhancedCode{5, 3, 4}, Message: "message too large for this moderated list"} case errors.Is(err, router.ErrNotMember): return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "only list members may post"} case errors.Is(err, router.ErrBadToken): diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index 1eecfa3..58135ac 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -289,3 +289,30 @@ func TestAFullModerationQueueAnswers452(t *testing.T) { t.Fatalf("a temporary refusal, so a legitimate sender retries later and a spam run goes away: %v", res["dev@example.org"]) } } + +func TestAMessageTooBigToHoldAnswers552(t *testing.T) { + st := newStack(t, nil) + l, _ := st.store.FindList("dev@example.org") + st.store.SetPolicies(l.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate) + conn, err := net.Dial("tcp", st.addr) + if err != nil { + t.Fatal(err) + } + c := smtp.NewClientLMTP(conn) + defer c.Close() + c.Hello("localhost") + c.Mail("alice@example.com", nil) + c.Rcpt("dev@example.org", nil) + w, _ := c.Data() + w.Write([]byte("From: Alice \r\nSubject: huge\r\n\r\n")) + line := []byte(strings.Repeat("x", 76) + "\r\n") + for i := 0; i < (10<<20)/len(line)+10; i++ { + w.Write(line) + } + _, err = w.CloseWithLMTPResponse() + var de smtp.LMTPDataError + var se *smtp.SMTPError + if !errors.As(err, &de) || !errors.As(de["dev@example.org"], &se) || se.Code != 552 { + t.Fatalf("a permanent refusal: the sender is told to send it elsewhere: %v", err) + } +} diff --git a/internal/router/identity_test.go b/internal/router/identity_test.go index 47c5c02..d4431ea 100644 --- a/internal/router/identity_test.go +++ b/internal/router/identity_test.go @@ -138,3 +138,16 @@ func TestSimultaneousApprovalsDeliverOnce(t *testing.T) { t.Fatalf("2 members, one approval: pending = %d", st.Pending) } } + +func TestAMessageTooBigToHoldIsRefused(t *testing.T) { + e := newEnv(t) + e.s.SetPolicies(e.list.ID, "moderated", "open", "private") + big := append(postFrom("alice@example.com"), make([]byte, 10<<20)...) + _, err := e.r.Route("alice@example.com", "dev@example.org", big) + if !errors.Is(err, ErrHeldTooLarge) { + t.Fatalf("want ErrHeldTooLarge, got %v", err) + } + if held, _ := e.s.ListHeld(e.list.ID); len(held) != 0 { + t.Fatal("nothing is held") + } +} diff --git a/internal/router/router.go b/internal/router/router.go index 1be4115..3f9c23b 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -22,6 +22,8 @@ var ( ErrDiscarded = errors.New("discarded") // ErrHeldFull means the list's moderation queue is full: refuse for now, the sender may retry. ErrHeldFull = store.ErrHeldFull + // ErrHeldTooLarge means the message is too big to hold for moderation on a moderated list. + ErrHeldTooLarge = store.ErrHeldTooLarge // ErrNotMember means the sender may not post to the list. ErrNotMember = errors.New("sender is not a member") // ErrHeld means the message was queued for moderation. It is not a failure. From bcca031ea3140ab9a121e326513bf253cf5b514d Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 11:40:31 +0100 Subject: [PATCH 4/6] feat: show who really wrote a held message in the CLI and the UI - Add SENDER and FROM columns to held ls, flagging a From header that differs from the sender - Show the From header and a badge in the Held tab; keep the envelope sender visible --- admin/src/lib/api.ts | 4 ++++ admin/src/pages/ListPage.test.tsx | 2 +- admin/src/tabs/HeldTab.test.tsx | 25 +++++++++++++++++++++++++ admin/src/tabs/HeldTab.tsx | 12 +++++++++++- internal/cli/cli_test.go | 29 ++++++++++++++++++++++++++++- internal/cli/commands.go | 13 +++++++++++-- 6 files changed, 80 insertions(+), 5 deletions(-) diff --git a/admin/src/lib/api.ts b/admin/src/lib/api.ts index eb5a5c8..0f25352 100644 --- a/admin/src/lib/api.ts +++ b/admin/src/lib/api.ts @@ -30,6 +30,10 @@ export interface Held { created_at: string /** Why the message was held, for the admin. */ reason: string + /** The From header as received: the author readers will see. */ + from: string + /** The From header names someone other than the envelope sender, or nobody usable. */ + mismatch: boolean } export interface Message { diff --git a/admin/src/pages/ListPage.test.tsx b/admin/src/pages/ListPage.test.tsx index be9710e..f2a8f11 100644 --- a/admin/src/pages/ListPage.test.tsx +++ b/admin/src/pages/ListPage.test.tsx @@ -50,7 +50,7 @@ describe('list page', () => { installFakeApi( newState({ lists: [dev()], - held: { 1: [{ id: 5, sender: 'a@x.com', subject: 's', size: 10, created_at: '2030-01-01T00:00:00Z', reason: '' }] }, + held: { 1: [{ id: 5, sender: 'a@x.com', subject: 's', size: 10, created_at: '2030-01-01T00:00:00Z', reason: '', from: 'a@x.com', mismatch: false }] }, }), ) renderApp('/lists/1', { token: 'good' }) diff --git a/admin/src/tabs/HeldTab.test.tsx b/admin/src/tabs/HeldTab.test.tsx index 3a0c169..5abdc84 100644 --- a/admin/src/tabs/HeldTab.test.tsx +++ b/admin/src/tabs/HeldTab.test.tsx @@ -11,6 +11,8 @@ const held = (id: number, over: Partial = {}): Held => ({ size: 2048, created_at: '2030-01-02T03:04:05Z', reason: 'the list is moderated', + from: `Sender ${id} `, + mismatch: false, ...over, }) @@ -46,6 +48,29 @@ describe('held tab', () => { expect(screen.getByRole('columnheader', { name: /reason/i })).toBeInTheDocument() }) + it('shows the author readers will see, and flags a From header that is not the sender', async () => { + await setup([ + held(1), + held(2, { sender: 'bob@example.com', from: 'Eve ', mismatch: true }), + held(3, { sender: 'bob@example.com', from: '', mismatch: true }), + ]) + const honest = (await screen.findByText('Subject 1')).closest('tr')! + expect(within(honest).getByText('Sender 1 ')).toBeInTheDocument() + expect(within(honest).getByText('s1@example.com')).toBeInTheDocument() + expect(within(honest).queryByText(/differs from the sender/i)).not.toBeInTheDocument() + + const passing = screen.getByText('Subject 2').closest('tr')! + expect(within(passing).getByText('Eve ')).toBeInTheDocument() + expect(within(passing).getByText('bob@example.com')).toBeInTheDocument() // who really sent it + expect(within(passing).getByText(/differs from the sender/i)).toBeInTheDocument() + + const noFrom = screen.getByText('Subject 3').closest('tr')! + expect(within(noFrom).getByText(/differs from the sender/i)).toBeInTheDocument() + expect(within(noFrom).getByText('(none)')).toBeInTheDocument() + expect(screen.getByRole('columnheader', { name: /^from \(header\)$/i })).toBeInTheDocument() + expect(screen.getByRole('columnheader', { name: /^sender$/i })).toBeInTheDocument() + }) + it('says nothing is waiting', async () => { await setup([]) expect(await screen.findByText(/nothing is waiting/i)).toBeInTheDocument() diff --git a/admin/src/tabs/HeldTab.tsx b/admin/src/tabs/HeldTab.tsx index bccaa48..fa3c0e6 100644 --- a/admin/src/tabs/HeldTab.tsx +++ b/admin/src/tabs/HeldTab.tsx @@ -11,6 +11,7 @@ import { AlertDialogHeader, AlertDialogTitle, } from '@/components/ui/alert-dialog' +import { Badge } from '@/components/ui/badge' import { Button } from '@/components/ui/button' import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table' import { api, type Held, type List } from '@/lib/api' @@ -60,7 +61,8 @@ export function HeldTab() { Subject - From + From (header) + Sender Received Size Reason @@ -71,6 +73,14 @@ export function HeldTab() { {items.map((h) => ( {h.subject || '(no subject)'} + + {h.from || (none)} + {h.mismatch && ( + + Differs from the sender + + )} + {h.sender} {formatDate(h.created_at)} {formatBytes(h.size)} diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index 05aca26..ff494f8 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -319,7 +319,7 @@ func TestHeldQueue(t *testing.T) { id2, _ := e.s.HoldMessage(l.ID, "eve@x.com", []byte("Subject: Second held\r\n\r\nbody"), "") id3, _ := e.s.HoldMessage(l.ID, "anon@x.com", []byte("From: anon@x.com\r\n\r\nno subject here"), "") out := e.ok(t, "held", "ls", "dev@example.org") - for _, want := range []string{"ID", "FROM", "SUBJECT", "SIZE", "REASON", "bob@x.com", "First held", "eve@x.com", "Second held", "(no subject)", "the list is moderated"} { + for _, want := range []string{"ID", "SENDER", "FROM", "SUBJECT", "SIZE", "REASON", "bob@x.com", "First held", "eve@x.com", "Second held", "(no subject)", "the list is moderated"} { if !strings.Contains(out, want) { t.Fatalf("held ls lacks %q:\n%s", want, out) } @@ -797,3 +797,30 @@ func TestQueueShowsFailedDeliveries(t *testing.T) { t.Fatalf("a failed delivery can be cleared:\n%s", out) } } + +func TestHeldListingWarnsWhenTheHeaderDiffersFromTheSender(t *testing.T) { + e := newEnv(t) + e.ok(t, "list", "add", "dev@example.org") + l, _ := e.s.FindList("dev@example.org") + e.s.HoldMessage(l.ID, "bob@x.com", []byte("From: Bob \r\nSubject: honest\r\n\r\nb"), "r") + e.s.HoldMessage(l.ID, "bob@x.com", []byte("From: Eve \r\nSubject: passing as eve\r\n\r\nb"), "r") + e.s.HoldMessage(l.ID, "bob@x.com", []byte("Subject: no from\r\n\r\nb"), "r") + out := e.ok(t, "held", "ls", "dev@example.org") + lines := map[string]string{} + for _, l := range strings.Split(out, "\n") { + for _, key := range []string{"honest", "passing as eve", "no from"} { + if strings.Contains(l, key) { + lines[key] = l + } + } + } + if strings.Contains(lines["honest"], "differs") { + t.Fatalf("a matching header is not flagged: %q", lines["honest"]) + } + if !strings.Contains(lines["passing as eve"], "Eve ") || !strings.Contains(lines["passing as eve"], "differs") { + t.Fatalf("the author readers will see is shown and flagged: %q", lines["passing as eve"]) + } + if !strings.Contains(lines["no from"], "differs") { + t.Fatalf("no usable From is flagged too: %q", lines["no from"]) + } +} diff --git a/internal/cli/commands.go b/internal/cli/commands.go index 0c31e02..0a09494 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -375,6 +375,8 @@ func (c *cmd) heldLs(args []string) int { Size int `json:"size"` CreatedAt time.Time `json:"created_at"` Reason string `json:"reason"` + From string `json:"from"` + Mismatch bool `json:"mismatch"` } if err := json.Unmarshal(raw, &hs); err != nil { return c.fail(err) @@ -383,13 +385,20 @@ func (c *cmd) heldLs(args []string) int { fmt.Fprintln(c.io.Out, "Nothing is held for moderation.") return 0 } - rows := [][]string{{"ID", "RECEIVED", "FROM", "SUBJECT", "SIZE", "REASON"}} + rows := [][]string{{"ID", "RECEIVED", "SENDER", "FROM", "SUBJECT", "SIZE", "REASON"}} for _, h := range hs { subject := h.Subject if subject == "" { subject = "(no subject)" } - rows = append(rows, []string{strconv.FormatInt(h.ID, 10), formatDate(h.CreatedAt), h.Sender, subject, formatBytes(h.Size), h.Reason}) + from := h.From + if from == "" { + from = "-" + } + if h.Mismatch { + from += " (differs)" // readers would see someone other than the sender + } + rows = append(rows, []string{strconv.FormatInt(h.ID, 10), formatDate(h.CreatedAt), h.Sender, from, subject, formatBytes(h.Size), h.Reason}) } c.table(rows) return 0 From beff2579c27524657c54cb90c794dc8b20f63b02 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 11:42:19 +0100 Subject: [PATCH 5/6] test: pin the SMTP command named in a relay error, SMTPUTF8 and thread order - Name the refused command in the error (MAIL FROM, RCPT TO, DATA) so the queue tells a refused sender from a refused recipient - Test a refusal of the DATA command itself, SMTPUTF8 for non-ASCII addresses and oldest-first threads --- internal/outbound/outbound.go | 9 ++- internal/outbound/outbound_test.go | 114 +++++++++++++++++++++++++++++ internal/store/archive_test.go | 24 ++++++ 3 files changed, 146 insertions(+), 1 deletion(-) diff --git a/internal/outbound/outbound.go b/internal/outbound/outbound.go index 95a0ef8..09ebcaa 100644 --- a/internal/outbound/outbound.go +++ b/internal/outbound/outbound.go @@ -279,7 +279,14 @@ type PhaseError struct { Err error } -func (e *PhaseError) Error() string { return e.Err.Error() } +// Error names the command, so the admin can tell a refused recipient from a refused sender. +func (e *PhaseError) Error() string { + label := map[string]string{"MAIL": "MAIL FROM", "RCPT": "RCPT TO", "DATA": "DATA"}[e.Phase] + if label == "" { + label = e.Phase + } + return label + ": " + e.Err.Error() +} func (e *PhaseError) Unwrap() error { return e.Err } // AtRcpt marks err as the relay's answer to RCPT TO (SMTPRelay does this itself; it is exported diff --git a/internal/outbound/outbound_test.go b/internal/outbound/outbound_test.go index a873fee..203dad5 100644 --- a/internal/outbound/outbound_test.go +++ b/internal/outbound/outbound_test.go @@ -220,6 +220,17 @@ func TestRelayRejectionsAreClassifiedByTheCommandTheyAnswer(t *testing.T) { if IsPermanent(err) != tc.perm || IsBadMailbox(err) != tc.mailbox { t.Errorf("%s: permanent=%v mailbox=%v for %v", name, IsPermanent(err), IsBadMailbox(err), err) } + // The error says which command was refused, for the admin reading the queue. + stage := map[bool]string{true: "RCPT TO: ", false: ""}[tc.r.rcpt != nil] + if tc.r.mail != nil { + stage = "MAIL FROM: " + } + if tc.r.data != nil { + stage = "DATA: " + } + if !strings.HasPrefix(err.Error(), stage) { + t.Errorf("%s: error %q should start with %q", name, err.Error(), stage) + } } } @@ -391,3 +402,106 @@ func TestSingleBatchErrorsAreNotDecoratedWithBatchNumbers(t *testing.T) { t.Fatalf("with several batches the failing range is useful: %v", err) } } + +func TestNonASCIIAddressesAskForSMTPUTF8(t *testing.T) { + var gotUTF8 []bool + var mu sync.Mutex + backend := smtp.BackendFunc(func(*smtp.Conn) (smtp.Session, error) { + return &utf8Session{record: func(v bool) { mu.Lock(); gotUTF8 = append(gotUTF8, v); mu.Unlock() }}, nil + }) + srv := smtp.NewServer(backend) + srv.EnableSMTPUTF8 = true + ln, _ := net.Listen("tcp", "127.0.0.1:0") + go srv.Serve(ln) + defer srv.Close() + r := &SMTPRelay{Addr: ln.Addr().String()} + if err := r.Send(context.Background(), "bounce@example.org", []string{"jörg@example.com"}, []byte("Subject: x\r\n\r\nx\r\n")); err != nil { + t.Fatal(err) + } + if err := r.Send(context.Background(), "bounce@example.org", []string{"plain@example.com"}, []byte("Subject: x\r\n\r\nx\r\n")); err != nil { + t.Fatal(err) + } + mu.Lock() + defer mu.Unlock() + if len(gotUTF8) != 2 || !gotUTF8[0] || gotUTF8[1] { + t.Fatalf("SMTPUTF8 only for the message that needs it: %v", gotUTF8) + } +} + +type utf8Session struct{ record func(bool) } + +func (s *utf8Session) Reset() {} +func (s *utf8Session) Logout() error { return nil } +func (s *utf8Session) Mail(_ string, o *smtp.MailOptions) error { + s.record(o != nil && o.UTF8) + return nil +} +func (s *utf8Session) Rcpt(string, *smtp.RcptOptions) error { return nil } +func (s *utf8Session) Data(r io.Reader) error { io.Copy(io.Discard, r); return nil } + +// refuseData is a bare SMTP server that accepts MAIL and RCPT but answers the DATA command itself with reply. +func refuseData(t *testing.T, reply string) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go func() { + defer c.Close() + c.Write([]byte("220 test ESMTP\r\n")) + one := make([]byte, 1) + var buf []byte + for { + if _, err := c.Read(one); err != nil { + return + } + buf = append(buf, one[0]) + if len(buf) < 2 || buf[len(buf)-2] != '\r' || buf[len(buf)-1] != '\n' { + continue + } + l := string(buf[:len(buf)-2]) + buf = buf[:0] + switch { + case strings.HasPrefix(l, "EHLO"), strings.HasPrefix(l, "HELO"): + c.Write([]byte("250 test\r\n")) + case strings.HasPrefix(l, "MAIL"), strings.HasPrefix(l, "RCPT"): + c.Write([]byte("250 2.1.0 ok\r\n")) + case l == "DATA": + c.Write([]byte(reply)) + case l == "QUIT": + c.Write([]byte("221 bye\r\n")) + return + } + } + }() + } + }() + return ln.Addr().String() +} + +func TestARefusalOfTheDATACommandItselfIsNotAMissingMailbox(t *testing.T) { + // 5.1.1 here would be absurd, but a hostile or broken relay must not be able to strike members with it. + for name, tc := range map[string]struct { + reply string + perm bool + }{ + "554 no valid recipients": {"554 5.5.1 no valid recipients\r\n", true}, + "5.1.1 at DATA": {"550 5.1.1 whatever\r\n", true}, + "451 try later": {"451 4.3.0 try later\r\n", false}, + } { + err := (&SMTPRelay{Addr: refuseData(t, tc.reply)}).Send(context.Background(), "b@x.org", []string{"a@x.com"}, []byte("Subject: x\r\n\r\nx\r\n")) + if err == nil || !strings.HasPrefix(err.Error(), "DATA: ") { + t.Fatalf("%s: the error says DATA was refused: %v", name, err) + } + if IsBadMailbox(err) || IsPermanent(err) != tc.perm { + t.Errorf("%s: permanent=%v mailbox=%v", name, IsPermanent(err), IsBadMailbox(err)) + } + } +} diff --git a/internal/store/archive_test.go b/internal/store/archive_test.go index e44b17f..e21e913 100644 --- a/internal/store/archive_test.go +++ b/internal/store/archive_test.go @@ -445,3 +445,27 @@ func TestThreadRootsFromTheNewestEndOfAChain(t *testing.T) { t.Fatalf("roots: %v", forward) } } + +func TestThreadListingsAreOldestFirst(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + late := msg("", "Late reply", "b") + late.Date = time.Unix(1_700_000_900, 0) + late.InReplyTo = "" + early := msg("", "Early reply", "b") + early.Date = time.Unix(1_700_000_100, 0) + early.InReplyTo = "" + root := msg("", "Root", "b") + root.Date = time.Unix(1_700_000_000, 0) + rootID, _, _ := s.ArchiveMessage(l.ID, root) + s.ArchiveMessage(l.ID, late) // inserted before the earlier reply + s.ArchiveMessage(l.ID, early) + th, _ := s.ThreadMessages(l.ID, rootID) + got := []string{} + for _, m := range th { + got = append(got, m.Subject) + } + if strings.Join(got, "|") != "Root|Early reply|Late reply" { + t.Fatalf("a thread reads in the order it happened, not in the order it was stored: %v", got) + } +} From 37241464a711c0c67bd94aef63db276d3647ba9a Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 11:42:42 +0100 Subject: [PATCH 6/6] docs: held queue summary, bounds and the From header in the API and README --- README.md | 4 ++++ docs/openapi.yaml | 7 +++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a6f52ba..61453bc 100644 --- a/README.md +++ b/README.md @@ -249,6 +249,10 @@ lists.example.org lmtp:inet:127.0.0.1:2424 and `Reply-To` come from the header; so a post whose header names someone else (or has no usable `From`, or several addresses) is held for moderation instead of delivered, with the reason shown to the admin (API, CLI, UI). Let Postfix reject forged senders too (SPF, DKIM, DMARC checks on inbound mail). +- **The moderation queue is bounded.** A list holds at most 500 messages and 200 MiB, and one held message at most 10 MiB; a + full queue answers `452` (the sender retries later), a message over 10 MiB `552`. Messages nobody moderated for 30 days + are dropped. Listings (API, CLI, UI) show a summary and never read the bodies. The held view shows the From header + readers would see next to the real sender and flags a difference. - **API token.** `parsimony serve` refuses a token shorter than 24 characters or containing whitespace (`openssl rand -hex 24` makes a good one). After 10 wrong or missing tokens within 5 minutes an address is locked out for 5 minutes (HTTP 429, valid token included). The address is the TCP peer: behind a reverse proxy diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 57a00b1..d5b334b 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -56,9 +56,12 @@ components: type: object properties: id: {type: integer} - sender: {type: string} - subject: {type: string} + sender: {type: string, description: The SMTP (envelope) sender} + from: {type: string, description: "The From header as received: the author readers will see (at most 200 characters)"} + mismatch: {type: boolean, description: The From header names someone other than the sender, or no usable address} + subject: {type: string, description: Decoded, at most 200 characters} size: {type: integer} + reason: {type: string, description: Why the message was held} created_at: {type: string, format: date-time} Error: type: object