fix(outbound): count a missing mailbox only when RCPT TO says so
- Issue MAIL, RCPT and DATA one by one so a failure knows which command it answers - Treat 5.1.1, 5.1.2, 5.1.3, 5.1.6 and 5.1.10 as 'no such mailbox' only in answer to RCPT TO - A refused sender (5.1.x answering MAIL FROM) or a refusal after DATA keeps the delivery as failed and never strikes members - Ignore a QUIT error once the relay accepted the message: it caused duplicate deliveries
This commit is contained in:
parent
f2eb79f56c
commit
c1c8211c1b
4 changed files with 239 additions and 27 deletions
|
|
@ -179,8 +179,11 @@ It answers `451` only when the database itself fails.
|
|||
- A permanent rejection of the message by the relay (5xx: policy, size, line length...) is never retried, but the
|
||||
delivery is not made to vanish: it is kept as **failed**, with the reason, for a week (queue API, `parsimony queue ls`,
|
||||
the Queue page; discard it to clear it). The same happens when a delivery is still failing after 3 days.
|
||||
- A bad-mailbox reply (`5.1.x`, the address does not exist) drops the delivery and counts as a hard bounce for the
|
||||
member (3 disable it). Other errors, such as "relay access denied", never disable members.
|
||||
- A reply to `RCPT TO` that says the address does not exist (`5.1.1`, `5.1.2`, `5.1.3`, `5.1.6`, `5.1.10`) drops the
|
||||
delivery and counts as a hard bounce for the member (3 disable it). Nothing else counts: not "relay access denied", not a
|
||||
refused sender (`5.1.7`, `5.1.8`, or any code answering `MAIL FROM`), not a refusal after `DATA`. Those deliveries are kept
|
||||
as failed.
|
||||
- A relay that accepted the message is never asked again because `QUIT` went wrong.
|
||||
- Delivery is at-least-once: a crash between the relay accepting a message and the queue entry being
|
||||
removed can send one message to one recipient twice.
|
||||
- `GET /api/v1/queue` and the Queue page show what is pending or failing, and a stuck delivery can be discarded.
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import (
|
|||
"crypto/x509"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
|
|
@ -176,10 +177,50 @@ func (r *SMTPRelay) sendBatch(ctx context.Context, from string, to []string, dat
|
|||
}
|
||||
}()
|
||||
|
||||
if err := c.SendMail(from, to, bytes.NewReader(data)); err != nil {
|
||||
return authError(err)
|
||||
// The commands are issued one by one, rather than with SendMail, to know which one failed.
|
||||
var mailOpts *smtp.MailOptions
|
||||
if !isASCII(from) || !allASCII(to) {
|
||||
mailOpts = &smtp.MailOptions{UTF8: true}
|
||||
}
|
||||
return c.Quit()
|
||||
if err := c.Mail(from, mailOpts); err != nil {
|
||||
return authError(&PhaseError{Phase: "MAIL", Err: err})
|
||||
}
|
||||
for _, rcpt := range to {
|
||||
if err := c.Rcpt(rcpt, nil); err != nil {
|
||||
return authError(&PhaseError{Phase: "RCPT", Err: err})
|
||||
}
|
||||
}
|
||||
w, err := c.Data()
|
||||
if err != nil {
|
||||
return authError(&PhaseError{Phase: "DATA", Err: err})
|
||||
}
|
||||
if _, err := io.Copy(w, bytes.NewReader(data)); err != nil {
|
||||
return &PhaseError{Phase: "DATA", Err: err}
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
return &PhaseError{Phase: "DATA", Err: err}
|
||||
}
|
||||
// The relay has accepted the message. If QUIT now fails, retrying would deliver it twice.
|
||||
c.Quit()
|
||||
return nil
|
||||
}
|
||||
|
||||
func isASCII(s string) bool {
|
||||
for i := 0; i < len(s); i++ {
|
||||
if s[i] >= 0x80 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func allASCII(ss []string) bool {
|
||||
for _, s := range ss {
|
||||
if !isASCII(s) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Sent is a message recorded by Fake.
|
||||
|
|
@ -230,15 +271,45 @@ func IsPermanent(err error) bool {
|
|||
return errors.As(err, &se) && se.Code >= 500 && se.Code <= 599
|
||||
}
|
||||
|
||||
// IsBadMailbox reports whether err says the destination mailbox is invalid (SMTP
|
||||
// enhanced status 5.1.x). Other permanent errors, such as "relay access denied",
|
||||
// say nothing about the recipient.
|
||||
// PhaseError says which SMTP command a failure answers. The same enhanced status means different
|
||||
// things depending on it: 5.1.1 to RCPT is "this mailbox does not exist", but to MAIL FROM it is
|
||||
// about our own sender address and says nothing about the recipient.
|
||||
type PhaseError struct {
|
||||
Phase string // "MAIL", "RCPT" or "DATA"
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *PhaseError) Error() string { return e.Err.Error() }
|
||||
func (e *PhaseError) Unwrap() error { return e.Err }
|
||||
|
||||
// AtRcpt marks err as the relay's answer to RCPT TO (SMTPRelay does this itself; it is exported
|
||||
// for tests that fake a relay).
|
||||
func AtRcpt(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return &PhaseError{Phase: "RCPT", Err: err}
|
||||
}
|
||||
|
||||
// recipientUnknown are the enhanced status codes that say the destination mailbox is invalid:
|
||||
// 5.1.1 bad mailbox, 5.1.2 bad domain, 5.1.3 bad address syntax, 5.1.6 moved, 5.1.10 null MX.
|
||||
var recipientUnknown = map[smtp.EnhancedCode]bool{
|
||||
{5, 1, 1}: true, {5, 1, 2}: true, {5, 1, 3}: true, {5, 1, 6}: true, {5, 1, 10}: true,
|
||||
}
|
||||
|
||||
// IsBadMailbox reports whether err is the relay saying the recipient does not exist: a 5.1.1, 5.1.2,
|
||||
// 5.1.3, 5.1.6 or 5.1.10 answer to RCPT TO. Anything else, such as "relay access denied", a policy
|
||||
// refusal or a rejected sender (5.1.7, 5.1.8, or any code answering MAIL FROM), says nothing about
|
||||
// the recipient and must not count against a member.
|
||||
func IsBadMailbox(err error) bool {
|
||||
var pe *PermanentError
|
||||
if errors.As(err, &pe) {
|
||||
return pe.BadMailbox
|
||||
}
|
||||
var ph *PhaseError
|
||||
if !errors.As(err, &ph) || ph.Phase != "RCPT" {
|
||||
return false
|
||||
}
|
||||
var se *smtp.SMTPError
|
||||
return errors.As(err, &se) && se.Code >= 500 && se.Code <= 599 &&
|
||||
se.EnhancedCode[0] == 5 && se.EnhancedCode[1] == 1
|
||||
return errors.As(err, &se) && se.Code >= 500 && se.Code <= 599 && recipientUnknown[se.EnhancedCode]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -145,13 +145,25 @@ func TestClassifyErrors(t *testing.T) {
|
|||
{"plain", errors.New("connection refused"), false, false},
|
||||
{"451", smtpErr(451, smtp.EnhancedCode{4, 3, 0}), false, false},
|
||||
{"421", smtpErr(421, smtp.NoEnhancedCode), false, false},
|
||||
{"550 unknown mailbox", smtpErr(550, smtp.EnhancedCode{5, 1, 1}), true, true},
|
||||
{"553 bad address syntax", smtpErr(553, smtp.EnhancedCode{5, 1, 3}), true, true},
|
||||
{"451 at RCPT", AtRcpt(smtpErr(451, smtp.EnhancedCode{4, 1, 1})), false, false},
|
||||
{"554 relay denied", smtpErr(554, smtp.EnhancedCode{5, 7, 1}), true, false},
|
||||
{"550 no enhanced code", smtpErr(550, smtp.NoEnhancedCode), true, false},
|
||||
{"wrapped", fmt.Errorf("batch 0-1: %w", smtpErr(550, smtp.EnhancedCode{5, 1, 1})), true, true},
|
||||
{"explicit permanent", &PermanentError{Err: errors.New("no")}, true, false},
|
||||
{"explicit bad mailbox", &PermanentError{Err: errors.New("no"), BadMailbox: true}, true, true},
|
||||
// The same enhanced code means different things depending on the command it answers.
|
||||
{"5.1.1 without a command", smtpErr(550, smtp.EnhancedCode{5, 1, 1}), true, false},
|
||||
{"5.1.1 at RCPT", AtRcpt(smtpErr(550, smtp.EnhancedCode{5, 1, 1})), true, true},
|
||||
{"5.1.2 at RCPT", AtRcpt(smtpErr(550, smtp.EnhancedCode{5, 1, 2})), true, true},
|
||||
{"5.1.3 at RCPT", AtRcpt(smtpErr(553, smtp.EnhancedCode{5, 1, 3})), true, true},
|
||||
{"5.1.6 at RCPT", AtRcpt(smtpErr(550, smtp.EnhancedCode{5, 1, 6})), true, true},
|
||||
{"5.1.10 at RCPT", AtRcpt(smtpErr(550, smtp.EnhancedCode{5, 1, 10})), true, true},
|
||||
{"wrapped, at RCPT", AtRcpt(fmt.Errorf("batch 0-1: %w", smtpErr(550, smtp.EnhancedCode{5, 1, 1}))), true, true},
|
||||
{"5.1.0 at RCPT", AtRcpt(smtpErr(550, smtp.EnhancedCode{5, 1, 0})), true, false},
|
||||
{"5.1.4 at RCPT", AtRcpt(smtpErr(550, smtp.EnhancedCode{5, 1, 4})), true, false},
|
||||
{"5.1.7 at RCPT (bad sender)", AtRcpt(smtpErr(553, smtp.EnhancedCode{5, 1, 7})), true, false},
|
||||
{"5.1.8 at RCPT (bad sender)", AtRcpt(smtpErr(553, smtp.EnhancedCode{5, 1, 8})), true, false},
|
||||
{"5.2.2 at RCPT (mailbox full)", AtRcpt(smtpErr(552, smtp.EnhancedCode{5, 2, 2})), true, false},
|
||||
{"5.7.1 at RCPT (policy)", AtRcpt(smtpErr(554, smtp.EnhancedCode{5, 7, 1})), true, false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := IsPermanent(c.err); got != c.perm {
|
||||
|
|
@ -163,28 +175,40 @@ func TestClassifyErrors(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
type rejecting struct{ err error }
|
||||
// rejecting refuses one stage of the SMTP transaction.
|
||||
type rejecting struct{ mail, rcpt, data error }
|
||||
|
||||
func (r rejecting) NewSession(*smtp.Conn) (smtp.Session, error) { return &rejectSession{r.err}, nil }
|
||||
func (r rejecting) NewSession(*smtp.Conn) (smtp.Session, error) { return &rejectSession{r}, nil }
|
||||
|
||||
type rejectSession struct{ err error }
|
||||
type rejectSession struct{ r rejecting }
|
||||
|
||||
func (s *rejectSession) Reset() {}
|
||||
func (s *rejectSession) Logout() error { return nil }
|
||||
func (s *rejectSession) Mail(string, *smtp.MailOptions) error { return nil }
|
||||
func (s *rejectSession) Rcpt(string, *smtp.RcptOptions) error { return s.err }
|
||||
func (s *rejectSession) Data(io.Reader) error { return nil }
|
||||
func (s *rejectSession) Mail(string, *smtp.MailOptions) error { return s.r.mail }
|
||||
func (s *rejectSession) Rcpt(string, *smtp.RcptOptions) error { return s.r.rcpt }
|
||||
func (s *rejectSession) Data(io.Reader) error { return s.r.data }
|
||||
|
||||
func TestRelayRejectionsAreClassifiedFromTheWire(t *testing.T) {
|
||||
func se(code int, enh smtp.EnhancedCode, msg string) error {
|
||||
return &smtp.SMTPError{Code: code, EnhancedCode: enh, Message: msg}
|
||||
}
|
||||
|
||||
func TestRelayRejectionsAreClassifiedByTheCommandTheyAnswer(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
err error
|
||||
r rejecting
|
||||
perm, mailbox bool
|
||||
}{
|
||||
"unknown user": {&smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"}, true, true},
|
||||
"relay denied": {&smtp.SMTPError{Code: 554, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "relay access denied"}, true, false},
|
||||
"try later": {&smtp.SMTPError{Code: 451, EnhancedCode: smtp.EnhancedCode{4, 3, 0}, Message: "try later"}, false, false},
|
||||
"unknown user at RCPT": {rejecting{rcpt: se(550, smtp.EnhancedCode{5, 1, 1}, "no such user")}, true, true},
|
||||
"relay denied at RCPT": {rejecting{rcpt: se(554, smtp.EnhancedCode{5, 7, 1}, "relay access denied")}, true, false},
|
||||
"try later at RCPT": {rejecting{rcpt: se(451, smtp.EnhancedCode{4, 3, 0}, "try later")}, false, false},
|
||||
"bad sender 5.1.8 at MAIL": {rejecting{mail: se(550, smtp.EnhancedCode{5, 1, 8}, "bad outbound sender")}, true, false},
|
||||
"sender domain 5.1.0 at MAIL": {rejecting{mail: se(550, smtp.EnhancedCode{5, 1, 0}, "sender domain not found")}, true, false},
|
||||
"unlisted sender 5.1.7 at MAIL": {rejecting{mail: se(553, smtp.EnhancedCode{5, 1, 7}, "sender address rejected")}, true, false},
|
||||
"even 5.1.1 at MAIL is the sender": {rejecting{mail: se(550, smtp.EnhancedCode{5, 1, 1}, "sender unknown")}, true, false},
|
||||
"5.1.1 after DATA is not a mailbox": {rejecting{data: se(550, smtp.EnhancedCode{5, 1, 1}, "content refused")}, true, false},
|
||||
"policy refusal after DATA": {rejecting{data: se(554, smtp.EnhancedCode{5, 7, 1}, "spam")}, true, false},
|
||||
"temporary after DATA": {rejecting{data: se(451, smtp.EnhancedCode{4, 3, 0}, "try later")}, false, false},
|
||||
} {
|
||||
srv := smtp.NewServer(rejecting{tc.err})
|
||||
srv := smtp.NewServer(tc.r)
|
||||
ln, _ := net.Listen("tcp", "127.0.0.1:0")
|
||||
go srv.Serve(ln)
|
||||
r := &SMTPRelay{Addr: ln.Addr().String()}
|
||||
|
|
@ -199,6 +223,94 @@ func TestRelayRejectionsAreClassifiedFromTheWire(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// quitDropper is a bare SMTP server that accepts the message and then hangs up on QUIT.
|
||||
type quitDropper struct {
|
||||
mu sync.Mutex
|
||||
accepted int
|
||||
}
|
||||
|
||||
func startQuitDropper(t *testing.T, reply string) (*quitDropper, string) {
|
||||
t.Helper()
|
||||
q := &quitDropper{}
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { ln.Close() })
|
||||
go func() {
|
||||
for {
|
||||
c, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
defer c.Close()
|
||||
c.Write([]byte("220 test ESMTP\r\n"))
|
||||
buf := make([]byte, 0, 4096)
|
||||
one := make([]byte, 1)
|
||||
inData := false
|
||||
line := func() (string, bool) {
|
||||
buf = buf[:0]
|
||||
for {
|
||||
if _, err := c.Read(one); err != nil {
|
||||
return "", false
|
||||
}
|
||||
buf = append(buf, one[0])
|
||||
if len(buf) >= 2 && buf[len(buf)-2] == '\r' && buf[len(buf)-1] == '\n' {
|
||||
return string(buf[:len(buf)-2]), true
|
||||
}
|
||||
}
|
||||
}
|
||||
for {
|
||||
l, ok := line()
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case inData:
|
||||
if l == "." {
|
||||
inData = false
|
||||
q.mu.Lock()
|
||||
q.accepted++
|
||||
q.mu.Unlock()
|
||||
c.Write([]byte("250 2.0.0 queued\r\n"))
|
||||
}
|
||||
case strings.HasPrefix(l, "EHLO"), strings.HasPrefix(l, "HELO"):
|
||||
c.Write([]byte("250 test\r\n"))
|
||||
case strings.HasPrefix(l, "MAIL"), strings.HasPrefix(l, "RCPT"):
|
||||
c.Write([]byte("250 2.1.0 ok\r\n"))
|
||||
case l == "DATA":
|
||||
inData = true
|
||||
c.Write([]byte("354 go ahead\r\n"))
|
||||
case l == "QUIT":
|
||||
if reply != "" {
|
||||
c.Write([]byte(reply))
|
||||
}
|
||||
return // hang up
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
}()
|
||||
return q, ln.Addr().String()
|
||||
}
|
||||
|
||||
func TestAFailedQuitAfterTheMessageWasAcceptedIsNotAFailure(t *testing.T) {
|
||||
// The relay said 250 to the message: retrying because QUIT misbehaved would deliver it twice.
|
||||
for name, reply := range map[string]string{"hang up without a reply": "", "421 instead of 221": "421 4.4.2 closing early\r\n"} {
|
||||
q, addr := startQuitDropper(t, reply)
|
||||
err := (&SMTPRelay{Addr: addr}).Send(context.Background(), "b@x.org", []string{"a@x.com"}, []byte("Subject: x\r\n\r\nx\r\n"))
|
||||
if err != nil {
|
||||
t.Errorf("%s: Send must succeed, the message was accepted: %v", name, err)
|
||||
}
|
||||
q.mu.Lock()
|
||||
if q.accepted != 1 {
|
||||
t.Errorf("%s: accepted %d times", name, q.accepted)
|
||||
}
|
||||
q.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// silent accepts connections and never says a word.
|
||||
func silent(t *testing.T) string {
|
||||
t.Helper()
|
||||
|
|
|
|||
|
|
@ -251,7 +251,7 @@ func TestBadMailboxCountsAHardBounceAndDisablesAfterThree(t *testing.T) {
|
|||
ms, _ := e.s.Members(l.ID)
|
||||
id := ms[0].ID
|
||||
e.snd.fn = func(context.Context, string, []string) error {
|
||||
return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"}
|
||||
return outbound.AtRcpt(&smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"})
|
||||
}
|
||||
for i := 1; i <= 3; i++ {
|
||||
e.s.Enqueue([]byte("Subject: x\r\n\r\nx"), []store.NewOutbound{{From: "b@x.org", Rcpt: "gone@x.com", MemberID: id}})
|
||||
|
|
@ -293,7 +293,7 @@ func TestOtherPermanentRejectionsNeverCountAgainstTheMember(t *testing.T) {
|
|||
func TestBadMailboxForSystemMailOrUnknownMemberIsHarmless(t *testing.T) {
|
||||
e := setup(t)
|
||||
e.snd.fn = func(context.Context, string, []string) error {
|
||||
return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"}
|
||||
return outbound.AtRcpt(&smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"})
|
||||
}
|
||||
e.s.Enqueue([]byte("x"), []store.NewOutbound{
|
||||
{From: "b@x.org", Rcpt: "a@x.com", MemberID: 0}, // confirmation mail
|
||||
|
|
@ -532,3 +532,29 @@ func TestEachAttemptIsSignedAgain(t *testing.T) {
|
|||
t.Fatalf("a retry is signed afresh (new timestamp), signing calls = %d", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARefusedSenderIsNotAMissingMailbox(t *testing.T) {
|
||||
// A relay that rejects the list's own sender address (5.1.8, 5.1.7, 5.1.0 or even 5.1.1 answering
|
||||
// MAIL FROM) says nothing about the recipients: the members must not be struck, and the mail must
|
||||
// stay visible as failed instead of vanishing.
|
||||
e := setup(t)
|
||||
l, _ := e.s.CreateList("dev", "example.org", "Dev")
|
||||
e.s.AddMember(l.ID, "ok@x.com", "")
|
||||
ms, _ := e.s.Members(l.ID)
|
||||
e.snd.fn = func(context.Context, string, []string) error {
|
||||
return &outbound.PhaseError{Phase: "MAIL", Err: &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 8}, Message: "Bad outbound sender"}}
|
||||
}
|
||||
for i := 0; i < 5; i++ {
|
||||
e.s.Enqueue([]byte("x"), []store.NewOutbound{{From: "dev+bounce-1-x@example.org", Rcpt: "ok@x.com", MemberID: ms[0].ID}})
|
||||
st, _ := e.w.RunOnce(context.Background())
|
||||
if st.Failed != 1 || st.Bounced != 0 || st.Dropped != 0 {
|
||||
t.Fatalf("round %d: %+v", i+1, st)
|
||||
}
|
||||
}
|
||||
if m, _ := e.s.MemberByID(ms[0].ID); m.BounceCount != 0 || m.Disabled {
|
||||
t.Fatalf("the member was struck for our own sender problem: %+v", m)
|
||||
}
|
||||
if stats, _ := e.s.OutboxStats(); stats.Failed != 5 {
|
||||
t.Fatalf("every delivery stays listed as failed: %+v", stats)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue