diff --git a/README.md b/README.md index 626accf..2bfe74a 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ A modern, single-binary mailing list manager: an evolution of Mailman. - Self-service subscribe and unsubscribe by email with double opt-in: send to `list+join@domain` or `list+leave@domain`, then reply to the confirmation. - Moderated lists: posts wait in a queue that the API can approve or discard. -- VERP bounces: every recipient gets a signed return address; 3 hard bounces disable a member. +- VERP bounces: every recipient gets a signed return address; 3 hard bounces within 30 days disable a member. - Archive with full-text search (SQLite FTS5) and threading. Each list has an `archive_policy`: `private` (default, API only), `public` (web pages) or `off`. Public pages hide email addresses. @@ -183,6 +183,12 @@ It answers `451` only when the database itself fails. delivery and counts as a hard bounce for the member (3 disable it). Nothing else counts: not "relay access denied", not a refused sender (`5.1.7`, `5.1.8`, or any code answering `MAIL FROM`), not a refusal after `DATA`. Those deliveries are kept as failed. +- A bounce that comes back later to the member's return address counts only if it is a real delivery status report + (RFC 3464: `multipart/report` with a `message/delivery-status` part) with `Action: failed` and one of the same + codes. A spam or policy rejection (`5.7.x`), a full mailbox (`5.2.x`), a delay, a `4.x.x`, or a `Status:` line in a + plain or quoted message does not count. +- Hard bounces count for 30 days: when the last one is older, the next one starts again from 1. A disabled member + stays disabled until it is added again. - A relay that accepted the message is never asked again because `QUIT` went wrong. - Delivery is at-least-once: a crash between the relay accepting a message and the queue entry being removed can send one message to one recipient twice. diff --git a/docs/openapi.yaml b/docs/openapi.yaml index d5b334b..a616aac 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -50,8 +50,8 @@ components: properties: address: {type: string} display_name: {type: string} - disabled: {type: boolean, description: Disabled after 3 hard bounces; re-adding the member re-enables it} - bounce_count: {type: integer} + disabled: {type: boolean, description: Disabled after 3 hard bounces within 30 days; re-adding the member re-enables it} + bounce_count: {type: integer, description: Recent hard bounces (restarts at 1 when the last one is older than 30 days)} Held: type: object properties: diff --git a/internal/router/bounce_test.go b/internal/router/bounce_test.go new file mode 100644 index 0000000..1f94419 --- /dev/null +++ b/internal/router/bounce_test.go @@ -0,0 +1,152 @@ +package router + +import ( + "fmt" + "strings" + "testing" +) + +// dsn builds a delivery status notification as an MTA sends it (RFC 3464): a multipart/report with +// a human part, a message/delivery-status part and the returned headers. +func dsn(action, status string) []byte { + return []byte(strings.ReplaceAll(fmt.Sprintf(`From: MAILER-DAEMON@mx.example.com +To: dev+bounce@example.org +Subject: Undelivered Mail Returned to Sender +MIME-Version: 1.0 +Content-Type: multipart/report; report-type=delivery-status; + boundary="BOUNDARY" + +--BOUNDARY +Content-Type: text/plain; charset=us-ascii + +I'm sorry to have to inform you that your message could not be delivered. + +--BOUNDARY +Content-Type: message/delivery-status + +Reporting-MTA: dns; mx.example.com +Arrival-Date: Mon, 9 Nov 2026 10:00:00 +0100 + +Final-Recipient: rfc822; alice@example.com +Original-Recipient: rfc822;alice@example.com +Action: %s +Status: %s +Diagnostic-Code: smtp; 550 something + +--BOUNDARY +Content-Type: text/rfc822-headers + +From: bob@example.com +Subject: hello + +--BOUNDARY-- +`, action, status), "\n", "\r\n")) +} + +func bounceCount(t *testing.T, e *env) int { + t.Helper() + m, err := e.s.MemberByID(e.alice) + if err != nil { + t.Fatal(err) + } + return m.BounceCount +} + +func TestOnlyFailedRecipientUnknownDSNsCount(t *testing.T) { + cases := []struct { + action, status string + counts bool + }{ + {"failed", "5.1.1", true}, + {"failed", "5.1.2", true}, + {"failed", "5.1.3", true}, + {"failed", "5.1.6", true}, + {"failed", "5.1.10", true}, + {"FAILED (permanent)", "5.1.1 (bad destination mailbox address)", true}, + {"failed", "5.7.1", false}, // spam or policy rejection of the content + {"failed", "5.2.2", false}, // mailbox full + {"failed", "5.1.7", false}, // bad sender address + {"failed", "5.1.11", false}, // not in the list (prefix of 5.1.1x must not match) + {"failed", "4.1.1", false}, + {"delayed", "5.1.1", false}, + {"delayed", "4.4.1", false}, + {"delivered", "2.0.0", false}, + } + for _, c := range cases { + e := newEnv(t) + handled, err := e.command("", e.r.bounceAddress(e.list, e.alice), dsn(c.action, c.status)) + if !handled || err != nil { + t.Fatalf("%s %s: %v %v", c.action, c.status, handled, err) + } + if got := bounceCount(t, e) == 1; got != c.counts { + t.Errorf("%s %s: counted=%v, want %v", c.action, c.status, got, c.counts) + } + } +} + +func TestStatusLinesOutsideADeliveryStatusPartDoNotCount(t *testing.T) { + quoted := "From: carol@example.com\r\nSubject: Re: your bounce\r\nContent-Type: text/plain\r\n\r\n" + + "> Action: failed\r\n> Status: 5.1.1\r\nAction: failed\r\nStatus: 5.1.1\r\n" + // A report whose status sits in the human part, not in message/delivery-status. + human := strings.Replace(string(dsn("delayed", "4.4.1")), "I'm sorry", "Action: failed\r\nStatus: 5.1.1\r\n\r\nI'm sorry", 1) + // The returned original message quotes a DSN of its own. + returned := strings.Replace(string(dsn("delayed", "4.4.1")), "Subject: hello", "Subject: hello\r\nAction: failed\r\nStatus: 5.1.1", 1) + for name, msg := range map[string]string{ + "plain text reply": quoted, + "status in human part": human, + "status in returned": returned, + "not mime at all": "Action: failed\r\nStatus: 5.1.1\r\n", + "broken multipart": "Content-Type: multipart/report; boundary=x\r\n\r\nno parts here\r\nStatus: 5.1.1\r\n", + "empty": "", + } { + e := newEnv(t) + if _, err := e.command("", e.r.bounceAddress(e.list, e.alice), []byte(msg)); err != nil { + t.Fatalf("%s: %v", name, err) + } + if n := bounceCount(t, e); n != 0 { + t.Errorf("%s: counted %d bounces", name, n) + } + } +} + +func TestAnyFailedRecipientBlockCounts(t *testing.T) { + // The first recipient block is a delay, the second a hard failure: the report counts once. + msg := strings.Replace(string(dsn("failed", "5.1.1")), "Final-Recipient:", + "Final-Recipient: rfc822; alice@example.com\r\nAction: delayed\r\nStatus: 4.4.1\r\n\r\nFinal-Recipient:", 1) + e := newEnv(t) + e.command("", e.r.bounceAddress(e.list, e.alice), []byte(msg)) + if n := bounceCount(t, e); n != 1 { + t.Fatalf("counted %d", n) + } +} + +func TestGlobalDeliveryStatusCounts(t *testing.T) { + // RFC 6533: the internationalized report has the same fields. + msg := strings.Replace(string(dsn("failed", "5.1.1")), "message/delivery-status", "message/global-delivery-status", 1) + e := newEnv(t) + e.command("", e.r.bounceAddress(e.list, e.alice), []byte(msg)) + if n := bounceCount(t, e); n != 1 { + t.Fatalf("counted %d", n) + } +} + +func TestOnlyTheFirstDeliveryStatusPartIsRead(t *testing.T) { + // A report has one delivery-status part; another one after it is not the MTA speaking. + extra := "--BOUNDARY\r\nContent-Type: message/delivery-status\r\n\r\nReporting-MTA: dns; x\r\n\r\n" + + "Final-Recipient: rfc822; alice@example.com\r\nAction: failed\r\nStatus: 5.1.1\r\n\r\n--BOUNDARY--" + msg := strings.Replace(string(dsn("delayed", "4.4.1")), "--BOUNDARY--", extra, 1) + e := newEnv(t) + e.command("", e.r.bounceAddress(e.list, e.alice), []byte(msg)) + if n := bounceCount(t, e); n != 0 { + t.Fatalf("counted %d", n) + } +} + +func TestAMalformedDeliveryStatusDoesNotCount(t *testing.T) { + msg := strings.Replace(string(dsn("failed", "5.1.1")), "Arrival-Date:", "this line is not a field\r\nArrival-Date:", 1) + e := newEnv(t) + e.command("", e.r.bounceAddress(e.list, e.alice), []byte(msg)) + if n := bounceCount(t, e); n != 0 { + t.Fatalf("counted %d", n) + } +} diff --git a/internal/router/commands.go b/internal/router/commands.go index 544a6a3..6ca741b 100644 --- a/internal/router/commands.go +++ b/internal/router/commands.go @@ -9,7 +9,6 @@ import ( "log" "mime" "net/mail" - "regexp" "strconv" "strings" "time" @@ -17,8 +16,6 @@ import ( "codeberg.org/squidd/parsimony/internal/store" ) -var statusRe = regexp.MustCompile(`(?mi)^Status:\s*([245])\.\d+\.\d+`) - func (r *Router) secret() ([]byte, error) { r.secretOnce.Do(func() { r.secretVal, r.secretErr = r.store.Secret() }) return r.secretVal, r.secretErr @@ -166,7 +163,7 @@ func (r *Router) bounce(l store.List, tag string, data []byte) error { if err != nil { return err } - if sm := statusRe.FindSubmatch(data); sm != nil && string(sm[1]) == "5" { + if isHardBounce(data) { _, err = r.store.RecordHardBounce(m.ID, store.BounceThreshold) } return err diff --git a/internal/router/dsn.go b/internal/router/dsn.go new file mode 100644 index 0000000..f121613 --- /dev/null +++ b/internal/router/dsn.go @@ -0,0 +1,66 @@ +package router + +import ( + "bufio" + "bytes" + "io" + "mime" + "mime/multipart" + "net/mail" + "net/textproto" + "strings" +) + +// recipientUnknown are the DSN status codes that say the recipient does not exist: 5.1.1 bad +// mailbox, 5.1.2 bad domain, 5.1.3 bad address syntax, 5.1.6 moved, 5.1.10 null MX. It is the +// same set as outbound.recipientUnknown, used for RCPT TO answers; keep the two in sync. +var recipientUnknown = map[string]bool{"5.1.1": true, "5.1.2": true, "5.1.3": true, "5.1.6": true, "5.1.10": true} + +// isHardBounce reports whether data is a delivery status notification (RFC 3464) that reports a +// recipient as non-existent: a multipart/report with a message/delivery-status part holding a +// per-recipient block with "Action: failed" and a status in recipientUnknown. Anything else (a +// policy or content rejection, a full mailbox, a delay, a status line in plain text) is not. +func isHardBounce(data []byte) bool { + msg, err := mail.ReadMessage(bytes.NewReader(data)) + if err != nil { + return false + } + mt, params, err := mime.ParseMediaType(msg.Header.Get("Content-Type")) + if err != nil || mt != "multipart/report" { + return false + } + mr := multipart.NewReader(msg.Body, params["boundary"]) + for { + p, err := mr.NextPart() + if err != nil { + return false + } + pt, _, _ := mime.ParseMediaType(p.Header.Get("Content-Type")) + if pt == "message/delivery-status" || pt == "message/global-delivery-status" { + return failedUnknownRecipient(p) + } + } +} + +// failedUnknownRecipient reads the fields of a delivery-status body: a per-message block, then one +// block per recipient, separated by blank lines. +func failedUnknownRecipient(r io.Reader) bool { + tr := textproto.NewReader(bufio.NewReader(r)) + for { + h, err := tr.ReadMIMEHeader() + if firstWord(h.Get("Action")) == "failed" && recipientUnknown[firstWord(h.Get("Status"))] { + return true + } + if err != nil { + return false + } + } +} + +// firstWord returns the lowercase value of a field without its trailing comment. +func firstWord(v string) string { + if f := strings.Fields(v); len(f) > 0 { + return strings.ToLower(f[0]) + } + return "" +} diff --git a/internal/router/m2_test.go b/internal/router/m2_test.go index 1a1d1c6..133206b 100644 --- a/internal/router/m2_test.go +++ b/internal/router/m2_test.go @@ -290,7 +290,7 @@ func TestUnknownCommandAndList(t *testing.T) { } func bounceMsg(status string) []byte { - return []byte(fmt.Sprintf("Subject: Undelivered Mail\r\nContent-Type: text/plain\r\n\r\nAction: failed\r\nStatus: %s\r\nDiagnostic-Code: smtp; 550 no such user\r\n", status)) + return dsn("failed", status) } func TestHardBouncesDisableMember(t *testing.T) { diff --git a/internal/store/bounce_test.go b/internal/store/bounce_test.go new file mode 100644 index 0000000..69212a8 --- /dev/null +++ b/internal/store/bounce_test.go @@ -0,0 +1,76 @@ +package store + +import ( + "testing" + "time" +) + +func TestHardBouncesCountOnlyWithinTheWindow(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_900_000_000, 0) + s.Now = func() time.Time { return now } + l, _ := s.CreateList("dev", "example.org", "Dev") + s.AddMember(l.ID, "a@x.com", "") + ms, _ := s.Members(l.ID) + id := ms[0].ID + count := func() int { + t.Helper() + m, err := s.MemberByID(id) + if err != nil { + t.Fatal(err) + } + return m.BounceCount + } + + s.RecordHardBounce(id, 3) + now = now.Add(BounceWindow) // exactly the window: still recent + s.RecordHardBounce(id, 3) + if n := count(); n != 2 { + t.Fatalf("a bounce within the window adds up: %d", n) + } + now = now.Add(BounceWindow + time.Second) // the last bounce is now too old + if disabled, err := s.RecordHardBounce(id, 3); disabled || err != nil { + t.Fatalf("an old count must not disable: %v %v", disabled, err) + } + if n := count(); n != 1 { + t.Fatalf("a bounce after a quiet window restarts the count: %d", n) + } + now = now.Add(time.Hour) + s.RecordHardBounce(id, 3) + if disabled, _ := s.RecordHardBounce(id, 3); !disabled { + t.Fatal("three recent bounces disable") + } + // A disabled member stays disabled, even when a late bounce restarts the count. + now = now.Add(2 * BounceWindow) + if disabled, _ := s.RecordHardBounce(id, 3); !disabled { + t.Fatal("only re-adding re-enables a member") + } +} + +func TestMigrationAddsTheBounceTimeColumn(t *testing.T) { + path := t.TempDir() + "/old.db" + s, _ := Open(path) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.AddMember(l.ID, "a@x.com", "") + s.db.Exec(`UPDATE members SET bounce_count=2`) + if _, err := s.db.Exec(`ALTER TABLE members DROP COLUMN bounce_at`); err != nil { + t.Fatal(err) + } + s.Close() + s2, err := Open(path) + if err != nil { + t.Fatalf("migrate: %v", err) + } + defer s2.Close() + ms, err := s2.Members(l.ID) + if err != nil || len(ms) != 1 || ms[0].BounceCount != 2 { + t.Fatalf("members survive the migration: %+v %v", ms, err) + } + // An old count has no date: it is older than the window, so the next bounce restarts it. + if disabled, err := s2.RecordHardBounce(ms[0].ID, 3); disabled || err != nil { + t.Fatalf("%v %v", disabled, err) + } + if m, _ := s2.MemberByID(ms[0].ID); m.BounceCount != 1 { + t.Fatalf("count: %d", m.BounceCount) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 5abaedf..0683169 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -58,6 +58,9 @@ var ( // BounceThreshold is the number of hard bounces after which a member is disabled. const BounceThreshold = 3 +// BounceWindow is how long a hard bounce counts: a bounce after a quiet window restarts the count. +const BounceWindow = 30 * 24 * time.Hour + const ( pendingTTL = 72 * time.Hour pendingRateLimit = time.Hour @@ -146,6 +149,7 @@ CREATE TABLE IF NOT EXISTS members ( display_name TEXT NOT NULL DEFAULT '', disabled INTEGER NOT NULL DEFAULT 0, bounce_count INTEGER NOT NULL DEFAULT 0, + bounce_at INTEGER NOT NULL DEFAULT 0, UNIQUE(list_id, address) ); CREATE TABLE IF NOT EXISTS held ( @@ -278,6 +282,15 @@ func migrate(db *sql.DB) error { } } } + if ok, err := hasColumn(db, "members", "id"); err != nil { + return err + } else if ok { + if has, _ := hasColumn(db, "members", "bounce_at"); !has { + if _, err := db.Exec(`ALTER TABLE members ADD COLUMN bounce_at INTEGER NOT NULL DEFAULT 0`); err != nil { + return err + } + } + } if ok, err := hasColumn(db, "outbox", "msg_id"); err != nil { return err } else if ok { @@ -545,9 +558,13 @@ func (s *Store) DeleteMember(id int64) error { } // RecordHardBounce counts a hard bounce and disables the member once the count -// reaches threshold. It reports whether the member is now disabled. +// reaches threshold. The count holds recent bounces only: when the last one is +// older than BounceWindow it restarts at 1. It reports whether the member is now disabled. func (s *Store) RecordHardBounce(memberID int64, threshold int) (bool, error) { - if err := affected(s.db.Exec(`UPDATE members SET bounce_count=bounce_count+1 WHERE id=?`, memberID)); err != nil { + now := s.Now() + if err := affected(s.db.Exec(`UPDATE members SET + bounce_count=CASE WHEN bounce_at=?`, memberID, threshold); err != nil {