Merge branch 'fix/bounce-accounting' into fix/fable-mediums
This commit is contained in:
commit
19c7dbb9c2
8 changed files with 324 additions and 10 deletions
|
|
@ -12,7 +12,7 @@ A modern, single-binary mailing list manager: an evolution of Mailman.
|
|||
- Self-service subscribe and unsubscribe by email with double opt-in:
|
||||
send to `list+join@domain` or `list+leave@domain`, then reply to the confirmation.
|
||||
- Moderated lists: posts wait in a queue that the API can approve or discard.
|
||||
- VERP bounces: every recipient gets a signed return address; 3 hard bounces disable a member.
|
||||
- VERP bounces: every recipient gets a signed return address; 3 hard bounces within 30 days disable a member.
|
||||
- Archive with full-text search (SQLite FTS5) and threading. Each list has an
|
||||
`archive_policy`: `private` (default, API only), `public` (web pages) or `off`.
|
||||
Public pages hide email addresses.
|
||||
|
|
@ -183,6 +183,12 @@ It answers `451` only when the database itself fails.
|
|||
delivery and counts as a hard bounce for the member (3 disable it). Nothing else counts: not "relay access denied", not a
|
||||
refused sender (`5.1.7`, `5.1.8`, or any code answering `MAIL FROM`), not a refusal after `DATA`. Those deliveries are kept
|
||||
as failed.
|
||||
- A bounce that comes back later to the member's return address counts only if it is a real delivery status report
|
||||
(RFC 3464: `multipart/report` with a `message/delivery-status` part) with `Action: failed` and one of the same
|
||||
codes. A spam or policy rejection (`5.7.x`), a full mailbox (`5.2.x`), a delay, a `4.x.x`, or a `Status:` line in a
|
||||
plain or quoted message does not count.
|
||||
- Hard bounces count for 30 days: when the last one is older, the next one starts again from 1. A disabled member
|
||||
stays disabled until it is added again.
|
||||
- A relay that accepted the message is never asked again because `QUIT` went wrong.
|
||||
- Delivery is at-least-once: a crash between the relay accepting a message and the queue entry being
|
||||
removed can send one message to one recipient twice.
|
||||
|
|
|
|||
|
|
@ -50,8 +50,8 @@ components:
|
|||
properties:
|
||||
address: {type: string}
|
||||
display_name: {type: string}
|
||||
disabled: {type: boolean, description: Disabled after 3 hard bounces; re-adding the member re-enables it}
|
||||
bounce_count: {type: integer}
|
||||
disabled: {type: boolean, description: Disabled after 3 hard bounces within 30 days; re-adding the member re-enables it}
|
||||
bounce_count: {type: integer, description: Recent hard bounces (restarts at 1 when the last one is older than 30 days)}
|
||||
Held:
|
||||
type: object
|
||||
properties:
|
||||
|
|
|
|||
152
internal/router/bounce_test.go
Normal file
152
internal/router/bounce_test.go
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
package router
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// dsn builds a delivery status notification as an MTA sends it (RFC 3464): a multipart/report with
|
||||
// a human part, a message/delivery-status part and the returned headers.
|
||||
func dsn(action, status string) []byte {
|
||||
return []byte(strings.ReplaceAll(fmt.Sprintf(`From: MAILER-DAEMON@mx.example.com
|
||||
To: dev+bounce@example.org
|
||||
Subject: Undelivered Mail Returned to Sender
|
||||
MIME-Version: 1.0
|
||||
Content-Type: multipart/report; report-type=delivery-status;
|
||||
boundary="BOUNDARY"
|
||||
|
||||
--BOUNDARY
|
||||
Content-Type: text/plain; charset=us-ascii
|
||||
|
||||
I'm sorry to have to inform you that your message could not be delivered.
|
||||
|
||||
--BOUNDARY
|
||||
Content-Type: message/delivery-status
|
||||
|
||||
Reporting-MTA: dns; mx.example.com
|
||||
Arrival-Date: Mon, 9 Nov 2026 10:00:00 +0100
|
||||
|
||||
Final-Recipient: rfc822; alice@example.com
|
||||
Original-Recipient: rfc822;alice@example.com
|
||||
Action: %s
|
||||
Status: %s
|
||||
Diagnostic-Code: smtp; 550 something
|
||||
|
||||
--BOUNDARY
|
||||
Content-Type: text/rfc822-headers
|
||||
|
||||
From: bob@example.com
|
||||
Subject: hello
|
||||
|
||||
--BOUNDARY--
|
||||
`, action, status), "\n", "\r\n"))
|
||||
}
|
||||
|
||||
func bounceCount(t *testing.T, e *env) int {
|
||||
t.Helper()
|
||||
m, err := e.s.MemberByID(e.alice)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m.BounceCount
|
||||
}
|
||||
|
||||
func TestOnlyFailedRecipientUnknownDSNsCount(t *testing.T) {
|
||||
cases := []struct {
|
||||
action, status string
|
||||
counts bool
|
||||
}{
|
||||
{"failed", "5.1.1", true},
|
||||
{"failed", "5.1.2", true},
|
||||
{"failed", "5.1.3", true},
|
||||
{"failed", "5.1.6", true},
|
||||
{"failed", "5.1.10", true},
|
||||
{"FAILED (permanent)", "5.1.1 (bad destination mailbox address)", true},
|
||||
{"failed", "5.7.1", false}, // spam or policy rejection of the content
|
||||
{"failed", "5.2.2", false}, // mailbox full
|
||||
{"failed", "5.1.7", false}, // bad sender address
|
||||
{"failed", "5.1.11", false}, // not in the list (prefix of 5.1.1x must not match)
|
||||
{"failed", "4.1.1", false},
|
||||
{"delayed", "5.1.1", false},
|
||||
{"delayed", "4.4.1", false},
|
||||
{"delivered", "2.0.0", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
e := newEnv(t)
|
||||
handled, err := e.command("", e.r.bounceAddress(e.list, e.alice), dsn(c.action, c.status))
|
||||
if !handled || err != nil {
|
||||
t.Fatalf("%s %s: %v %v", c.action, c.status, handled, err)
|
||||
}
|
||||
if got := bounceCount(t, e) == 1; got != c.counts {
|
||||
t.Errorf("%s %s: counted=%v, want %v", c.action, c.status, got, c.counts)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusLinesOutsideADeliveryStatusPartDoNotCount(t *testing.T) {
|
||||
quoted := "From: carol@example.com\r\nSubject: Re: your bounce\r\nContent-Type: text/plain\r\n\r\n" +
|
||||
"> Action: failed\r\n> Status: 5.1.1\r\nAction: failed\r\nStatus: 5.1.1\r\n"
|
||||
// A report whose status sits in the human part, not in message/delivery-status.
|
||||
human := strings.Replace(string(dsn("delayed", "4.4.1")), "I'm sorry", "Action: failed\r\nStatus: 5.1.1\r\n\r\nI'm sorry", 1)
|
||||
// The returned original message quotes a DSN of its own.
|
||||
returned := strings.Replace(string(dsn("delayed", "4.4.1")), "Subject: hello", "Subject: hello\r\nAction: failed\r\nStatus: 5.1.1", 1)
|
||||
for name, msg := range map[string]string{
|
||||
"plain text reply": quoted,
|
||||
"status in human part": human,
|
||||
"status in returned": returned,
|
||||
"not mime at all": "Action: failed\r\nStatus: 5.1.1\r\n",
|
||||
"broken multipart": "Content-Type: multipart/report; boundary=x\r\n\r\nno parts here\r\nStatus: 5.1.1\r\n",
|
||||
"empty": "",
|
||||
} {
|
||||
e := newEnv(t)
|
||||
if _, err := e.command("", e.r.bounceAddress(e.list, e.alice), []byte(msg)); err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
if n := bounceCount(t, e); n != 0 {
|
||||
t.Errorf("%s: counted %d bounces", name, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnyFailedRecipientBlockCounts(t *testing.T) {
|
||||
// The first recipient block is a delay, the second a hard failure: the report counts once.
|
||||
msg := strings.Replace(string(dsn("failed", "5.1.1")), "Final-Recipient:",
|
||||
"Final-Recipient: rfc822; alice@example.com\r\nAction: delayed\r\nStatus: 4.4.1\r\n\r\nFinal-Recipient:", 1)
|
||||
e := newEnv(t)
|
||||
e.command("", e.r.bounceAddress(e.list, e.alice), []byte(msg))
|
||||
if n := bounceCount(t, e); n != 1 {
|
||||
t.Fatalf("counted %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGlobalDeliveryStatusCounts(t *testing.T) {
|
||||
// RFC 6533: the internationalized report has the same fields.
|
||||
msg := strings.Replace(string(dsn("failed", "5.1.1")), "message/delivery-status", "message/global-delivery-status", 1)
|
||||
e := newEnv(t)
|
||||
e.command("", e.r.bounceAddress(e.list, e.alice), []byte(msg))
|
||||
if n := bounceCount(t, e); n != 1 {
|
||||
t.Fatalf("counted %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyTheFirstDeliveryStatusPartIsRead(t *testing.T) {
|
||||
// A report has one delivery-status part; another one after it is not the MTA speaking.
|
||||
extra := "--BOUNDARY\r\nContent-Type: message/delivery-status\r\n\r\nReporting-MTA: dns; x\r\n\r\n" +
|
||||
"Final-Recipient: rfc822; alice@example.com\r\nAction: failed\r\nStatus: 5.1.1\r\n\r\n--BOUNDARY--"
|
||||
msg := strings.Replace(string(dsn("delayed", "4.4.1")), "--BOUNDARY--", extra, 1)
|
||||
e := newEnv(t)
|
||||
e.command("", e.r.bounceAddress(e.list, e.alice), []byte(msg))
|
||||
if n := bounceCount(t, e); n != 0 {
|
||||
t.Fatalf("counted %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMalformedDeliveryStatusDoesNotCount(t *testing.T) {
|
||||
msg := strings.Replace(string(dsn("failed", "5.1.1")), "Arrival-Date:", "this line is not a field\r\nArrival-Date:", 1)
|
||||
e := newEnv(t)
|
||||
e.command("", e.r.bounceAddress(e.list, e.alice), []byte(msg))
|
||||
if n := bounceCount(t, e); n != 0 {
|
||||
t.Fatalf("counted %d", n)
|
||||
}
|
||||
}
|
||||
|
|
@ -9,7 +9,6 @@ import (
|
|||
"log"
|
||||
"mime"
|
||||
"net/mail"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
|
@ -17,8 +16,6 @@ import (
|
|||
"codeberg.org/squidd/parsimony/internal/store"
|
||||
)
|
||||
|
||||
var statusRe = regexp.MustCompile(`(?mi)^Status:\s*([245])\.\d+\.\d+`)
|
||||
|
||||
func (r *Router) secret() ([]byte, error) {
|
||||
r.secretOnce.Do(func() { r.secretVal, r.secretErr = r.store.Secret() })
|
||||
return r.secretVal, r.secretErr
|
||||
|
|
@ -166,7 +163,7 @@ func (r *Router) bounce(l store.List, tag string, data []byte) error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if sm := statusRe.FindSubmatch(data); sm != nil && string(sm[1]) == "5" {
|
||||
if isHardBounce(data) {
|
||||
_, err = r.store.RecordHardBounce(m.ID, store.BounceThreshold)
|
||||
}
|
||||
return err
|
||||
|
|
|
|||
66
internal/router/dsn.go
Normal file
66
internal/router/dsn.go
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
package router
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"io"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"net/mail"
|
||||
"net/textproto"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// recipientUnknown are the DSN status codes that say the recipient does not exist: 5.1.1 bad
|
||||
// mailbox, 5.1.2 bad domain, 5.1.3 bad address syntax, 5.1.6 moved, 5.1.10 null MX. It is the
|
||||
// same set as outbound.recipientUnknown, used for RCPT TO answers; keep the two in sync.
|
||||
var recipientUnknown = map[string]bool{"5.1.1": true, "5.1.2": true, "5.1.3": true, "5.1.6": true, "5.1.10": true}
|
||||
|
||||
// isHardBounce reports whether data is a delivery status notification (RFC 3464) that reports a
|
||||
// recipient as non-existent: a multipart/report with a message/delivery-status part holding a
|
||||
// per-recipient block with "Action: failed" and a status in recipientUnknown. Anything else (a
|
||||
// policy or content rejection, a full mailbox, a delay, a status line in plain text) is not.
|
||||
func isHardBounce(data []byte) bool {
|
||||
msg, err := mail.ReadMessage(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
mt, params, err := mime.ParseMediaType(msg.Header.Get("Content-Type"))
|
||||
if err != nil || mt != "multipart/report" {
|
||||
return false
|
||||
}
|
||||
mr := multipart.NewReader(msg.Body, params["boundary"])
|
||||
for {
|
||||
p, err := mr.NextPart()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
pt, _, _ := mime.ParseMediaType(p.Header.Get("Content-Type"))
|
||||
if pt == "message/delivery-status" || pt == "message/global-delivery-status" {
|
||||
return failedUnknownRecipient(p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// failedUnknownRecipient reads the fields of a delivery-status body: a per-message block, then one
|
||||
// block per recipient, separated by blank lines.
|
||||
func failedUnknownRecipient(r io.Reader) bool {
|
||||
tr := textproto.NewReader(bufio.NewReader(r))
|
||||
for {
|
||||
h, err := tr.ReadMIMEHeader()
|
||||
if firstWord(h.Get("Action")) == "failed" && recipientUnknown[firstWord(h.Get("Status"))] {
|
||||
return true
|
||||
}
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// firstWord returns the lowercase value of a field without its trailing comment.
|
||||
func firstWord(v string) string {
|
||||
if f := strings.Fields(v); len(f) > 0 {
|
||||
return strings.ToLower(f[0])
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
|
@ -290,7 +290,7 @@ func TestUnknownCommandAndList(t *testing.T) {
|
|||
}
|
||||
|
||||
func bounceMsg(status string) []byte {
|
||||
return []byte(fmt.Sprintf("Subject: Undelivered Mail\r\nContent-Type: text/plain\r\n\r\nAction: failed\r\nStatus: %s\r\nDiagnostic-Code: smtp; 550 no such user\r\n", status))
|
||||
return dsn("failed", status)
|
||||
}
|
||||
|
||||
func TestHardBouncesDisableMember(t *testing.T) {
|
||||
|
|
|
|||
76
internal/store/bounce_test.go
Normal file
76
internal/store/bounce_test.go
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
package store
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestHardBouncesCountOnlyWithinTheWindow(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
now := time.Unix(1_900_000_000, 0)
|
||||
s.Now = func() time.Time { return now }
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
s.AddMember(l.ID, "a@x.com", "")
|
||||
ms, _ := s.Members(l.ID)
|
||||
id := ms[0].ID
|
||||
count := func() int {
|
||||
t.Helper()
|
||||
m, err := s.MemberByID(id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m.BounceCount
|
||||
}
|
||||
|
||||
s.RecordHardBounce(id, 3)
|
||||
now = now.Add(BounceWindow) // exactly the window: still recent
|
||||
s.RecordHardBounce(id, 3)
|
||||
if n := count(); n != 2 {
|
||||
t.Fatalf("a bounce within the window adds up: %d", n)
|
||||
}
|
||||
now = now.Add(BounceWindow + time.Second) // the last bounce is now too old
|
||||
if disabled, err := s.RecordHardBounce(id, 3); disabled || err != nil {
|
||||
t.Fatalf("an old count must not disable: %v %v", disabled, err)
|
||||
}
|
||||
if n := count(); n != 1 {
|
||||
t.Fatalf("a bounce after a quiet window restarts the count: %d", n)
|
||||
}
|
||||
now = now.Add(time.Hour)
|
||||
s.RecordHardBounce(id, 3)
|
||||
if disabled, _ := s.RecordHardBounce(id, 3); !disabled {
|
||||
t.Fatal("three recent bounces disable")
|
||||
}
|
||||
// A disabled member stays disabled, even when a late bounce restarts the count.
|
||||
now = now.Add(2 * BounceWindow)
|
||||
if disabled, _ := s.RecordHardBounce(id, 3); !disabled {
|
||||
t.Fatal("only re-adding re-enables a member")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrationAddsTheBounceTimeColumn(t *testing.T) {
|
||||
path := t.TempDir() + "/old.db"
|
||||
s, _ := Open(path)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
s.AddMember(l.ID, "a@x.com", "")
|
||||
s.db.Exec(`UPDATE members SET bounce_count=2`)
|
||||
if _, err := s.db.Exec(`ALTER TABLE members DROP COLUMN bounce_at`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.Close()
|
||||
s2, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
defer s2.Close()
|
||||
ms, err := s2.Members(l.ID)
|
||||
if err != nil || len(ms) != 1 || ms[0].BounceCount != 2 {
|
||||
t.Fatalf("members survive the migration: %+v %v", ms, err)
|
||||
}
|
||||
// An old count has no date: it is older than the window, so the next bounce restarts it.
|
||||
if disabled, err := s2.RecordHardBounce(ms[0].ID, 3); disabled || err != nil {
|
||||
t.Fatalf("%v %v", disabled, err)
|
||||
}
|
||||
if m, _ := s2.MemberByID(ms[0].ID); m.BounceCount != 1 {
|
||||
t.Fatalf("count: %d", m.BounceCount)
|
||||
}
|
||||
}
|
||||
|
|
@ -58,6 +58,9 @@ var (
|
|||
// BounceThreshold is the number of hard bounces after which a member is disabled.
|
||||
const BounceThreshold = 3
|
||||
|
||||
// BounceWindow is how long a hard bounce counts: a bounce after a quiet window restarts the count.
|
||||
const BounceWindow = 30 * 24 * time.Hour
|
||||
|
||||
const (
|
||||
pendingTTL = 72 * time.Hour
|
||||
pendingRateLimit = time.Hour
|
||||
|
|
@ -146,6 +149,7 @@ CREATE TABLE IF NOT EXISTS members (
|
|||
display_name TEXT NOT NULL DEFAULT '',
|
||||
disabled INTEGER NOT NULL DEFAULT 0,
|
||||
bounce_count INTEGER NOT NULL DEFAULT 0,
|
||||
bounce_at INTEGER NOT NULL DEFAULT 0,
|
||||
UNIQUE(list_id, address)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS held (
|
||||
|
|
@ -278,6 +282,15 @@ func migrate(db *sql.DB) error {
|
|||
}
|
||||
}
|
||||
}
|
||||
if ok, err := hasColumn(db, "members", "id"); err != nil {
|
||||
return err
|
||||
} else if ok {
|
||||
if has, _ := hasColumn(db, "members", "bounce_at"); !has {
|
||||
if _, err := db.Exec(`ALTER TABLE members ADD COLUMN bounce_at INTEGER NOT NULL DEFAULT 0`); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if ok, err := hasColumn(db, "outbox", "msg_id"); err != nil {
|
||||
return err
|
||||
} else if ok {
|
||||
|
|
@ -545,9 +558,13 @@ func (s *Store) DeleteMember(id int64) error {
|
|||
}
|
||||
|
||||
// RecordHardBounce counts a hard bounce and disables the member once the count
|
||||
// reaches threshold. It reports whether the member is now disabled.
|
||||
// reaches threshold. The count holds recent bounces only: when the last one is
|
||||
// older than BounceWindow it restarts at 1. It reports whether the member is now disabled.
|
||||
func (s *Store) RecordHardBounce(memberID int64, threshold int) (bool, error) {
|
||||
if err := affected(s.db.Exec(`UPDATE members SET bounce_count=bounce_count+1 WHERE id=?`, memberID)); err != nil {
|
||||
now := s.Now()
|
||||
if err := affected(s.db.Exec(`UPDATE members SET
|
||||
bounce_count=CASE WHEN bounce_at<? THEN 1 ELSE bounce_count+1 END, bounce_at=? WHERE id=?`,
|
||||
now.Add(-BounceWindow).Unix(), now.Unix(), memberID)); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if _, err := s.db.Exec(`UPDATE members SET disabled=1 WHERE id=? AND bounce_count>=?`, memberID, threshold); err != nil {
|
||||
|
|
|
|||
Loading…
Reference in a new issue