feat(router): queue a post to a list only once per hour

When the reply to a delivered post is lost, the MTA sends the post again
and every member got it twice. The list now remembers the posts it queued.

- store: new delivered table (list id, post id, time), created with
  CREATE TABLE IF NOT EXISTS and dropped with its list
- store: EnqueuePost records the post and queues it in one transaction,
  returns ErrDuplicate when the list queued it in the last hour, and
  purges older records
- store: EnqueueFromHeld records the post too, without refusing it: an
  approval always delivers
- router: Deliver uses EnqueuePost and logs a duplicate; the post id is
  the Message-ID, or a SHA-256 of the message when it has none
- router: Release passes the post id to EnqueueFromHeld
- tests for duplicates, other ids and lists, expiry on the store clock,
  atomicity, posts without Message-ID and approvals
This commit is contained in:
tgiachi 2026-11-10 11:52:25 +01:00
commit bd4258ebac
6 changed files with 340 additions and 9 deletions

108
internal/router/dup_test.go Normal file
View file

@ -0,0 +1,108 @@
package router
import (
"errors"
"testing"
)
func pending(t *testing.T, e *env) int {
t.Helper()
st, err := e.s.OutboxStats()
if err != nil {
t.Fatal(err)
}
return st.Pending
}
func (e *env) post(t *testing.T, msg string) error {
t.Helper()
res, err := e.r.Route("alice@example.com", "dev@example.org", []byte(msg))
if err != nil {
t.Fatalf("route: %v", err)
}
return e.r.Deliver(res)
}
func TestTheSamePostIsDeliveredOnce(t *testing.T) {
e := newEnv(t)
woke := 0
e.r.OnEnqueue = func() { woke++ }
msg := "From: alice@example.com\r\nSubject: hi\r\nMessage-ID: <one@x>\r\n\r\nbody\r\n"
if err := e.post(t, msg); err != nil {
t.Fatal(err)
}
if err := e.post(t, msg); !errors.Is(err, ErrDuplicate) {
t.Fatalf("want ErrDuplicate, got %v", err)
}
if n := pending(t, e); n != 2 {
t.Fatalf("pending = %d, want one delivery per member", n)
}
if woke != 1 {
t.Fatalf("OnEnqueue called %d times for a duplicate", woke)
}
}
func TestAnotherMessageIDIsAnotherPost(t *testing.T) {
e := newEnv(t)
if err := e.post(t, "From: alice@example.com\r\nMessage-ID: <one@x>\r\n\r\nbody\r\n"); err != nil {
t.Fatal(err)
}
if err := e.post(t, "From: alice@example.com\r\nMessage-ID: <two@x>\r\n\r\nbody\r\n"); err != nil {
t.Fatal(err)
}
if n := pending(t, e); n != 4 {
t.Fatalf("pending = %d", n)
}
}
func TestAPostWithoutMessageIDIsKnownByItsContent(t *testing.T) {
e := newEnv(t)
msg := "From: alice@example.com\r\nSubject: hi\r\n\r\nbody\r\n"
if err := e.post(t, msg); err != nil {
t.Fatal(err)
}
if err := e.post(t, msg); !errors.Is(err, ErrDuplicate) {
t.Fatalf("the same bytes again: want ErrDuplicate, got %v", err)
}
if err := e.post(t, msg+"more\r\n"); err != nil {
t.Fatalf("another body is another post: %v", err)
}
if n := pending(t, e); n != 4 {
t.Fatalf("pending = %d", n)
}
}
func TestPostIDIgnoresTheRestWhenThereIsAMessageID(t *testing.T) {
a := postID([]byte("Message-ID: <one@x> \r\nSubject: a\r\n\r\nx"))
b := postID([]byte("Subject: b\r\nMessage-ID: <one@x>\r\n\r\ny"))
if a != b || a != "mid:<one@x>" {
t.Fatalf("%q %q", a, b)
}
if c := postID([]byte("Subject: b\r\n\r\ny")); c == postID([]byte("Subject: b\r\n\r\nz")) || len(c) < 10 {
t.Fatalf("content hash: %q", c)
}
if postID([]byte("Message-ID: <one@x>\r\n\r\n")) == postID([]byte("Message-ID: <One@x>\r\n\r\n")) {
t.Fatal("Message-IDs are case sensitive")
}
}
func TestAnApprovedPostIsDeliveredEvenIfItWasSeen(t *testing.T) {
e := newEnv(t)
msg := "From: alice@example.com\r\nSubject: hi\r\nMessage-ID: <one@x>\r\n\r\nbody\r\n"
if err := e.post(t, msg); err != nil {
t.Fatal(err)
}
id, err := e.s.HoldMessage(e.list.ID, "alice@example.com", []byte(msg), "r")
if err != nil {
t.Fatal(err)
}
if err := e.r.Release(id); err != nil {
t.Fatalf("release: %v", err)
}
if n := pending(t, e); n != 4 {
t.Fatalf("pending = %d, the approval must deliver", n)
}
if err := e.post(t, msg); !errors.Is(err, ErrDuplicate) {
t.Fatalf("a copy after the approval: want ErrDuplicate, got %v", err)
}
}

View file

@ -3,6 +3,8 @@ package router
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"errors"
"log"
"net/mail"
@ -34,6 +36,9 @@ var (
ErrGone = errors.New("already unsubscribed")
// ErrJoinClosed means the list does not accept self-service subscriptions.
ErrJoinClosed = errors.New("list is closed")
// ErrDuplicate means the list already queued the same post less than an hour ago: the MTA sent
// it again because our reply was lost. It is acknowledged and not queued again.
ErrDuplicate = store.ErrDuplicate
)
// Recipient is a subscriber that receives a message.
@ -126,13 +131,36 @@ func (r *Router) prepare(l store.List, data []byte) (Result, error) {
// Deliver queues the message for every recipient, each with its own signed
// bounce address (VERP) and unsubscribe headers. The queue write is atomic: on
// error nothing was queued and the caller may safely retry.
// error nothing was queued and the caller may safely retry. A post the list
// already queued in the last hour is not queued again: Deliver returns ErrDuplicate.
func (r *Router) Deliver(res Result) error {
items, err := r.outbound(res)
if err != nil {
return err
}
return r.enqueue(res.Data, items)
id := postID(res.Data)
if err := r.store.EnqueuePost(res.List.ID, id, res.Data, items); err != nil {
if errors.Is(err, ErrDuplicate) {
log.Printf("duplicate post to %s not delivered again: %s", res.List.Address(), id)
}
return err
}
if r.OnEnqueue != nil && len(items) > 0 {
r.OnEnqueue()
}
return nil
}
// postID identifies a post for the duplicate check: its Message-ID, or a hash of the whole
// message when it has none.
func postID(data []byte) string {
if m, err := mail.ReadMessage(bytes.NewReader(data)); err == nil {
if id := strings.TrimSpace(m.Header.Get("Message-ID")); id != "" {
return "mid:" + id
}
}
sum := sha256.Sum256(data)
return "sha256:" + hex.EncodeToString(sum[:])
}
// outbound builds one queue entry per recipient.
@ -213,7 +241,7 @@ func (r *Router) Release(heldID int64) error {
if err != nil {
return err
}
if err := r.store.EnqueueFromHeld(heldID, res.Data, items); err != nil {
if err := r.store.EnqueueFromHeld(heldID, postID(res.Data), res.Data, items); err != nil {
return err
}
if r.OnEnqueue != nil && len(items) > 0 {

145
internal/store/dup_test.go Normal file
View file

@ -0,0 +1,145 @@
package store
import (
"errors"
"testing"
"time"
)
func TestEnqueuePostRefusesTheSamePostTwiceWithinAnHour(t *testing.T) {
s := newTestStore(t)
now := time.Unix(1_000_000, 0)
s.Now = func() time.Time { return now }
l, _ := s.CreateList("dev", "example.org", "Dev")
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte("m"), items(2)); err != nil {
t.Fatal(err)
}
now = now.Add(59 * time.Minute)
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte("m"), items(2)); !errors.Is(err, ErrDuplicate) {
t.Fatalf("want ErrDuplicate, got %v", err)
}
if st, _ := s.OutboxStats(); st.Pending != 2 {
t.Fatalf("a duplicate must queue nothing, pending = %d", st.Pending)
}
}
func TestEnqueuePostTellsPostsAndListsApart(t *testing.T) {
s := newTestStore(t)
dev, _ := s.CreateList("dev", "example.org", "Dev")
ops, _ := s.CreateList("ops", "example.org", "Ops")
if err := s.EnqueuePost(dev.ID, "mid:<a@x>", []byte("m"), items(1)); err != nil {
t.Fatal(err)
}
if err := s.EnqueuePost(dev.ID, "mid:<b@x>", []byte("m"), items(1)); err != nil {
t.Fatalf("another Message-ID: %v", err)
}
if err := s.EnqueuePost(ops.ID, "mid:<a@x>", []byte("m"), items(1)); err != nil {
t.Fatalf("the same post to another list: %v", err)
}
if st, _ := s.OutboxStats(); st.Pending != 3 {
t.Fatalf("pending = %d", st.Pending)
}
}
func TestEnqueuePostForgetsAPostAfterAnHour(t *testing.T) {
s := newTestStore(t)
now := time.Unix(1_000_000, 0)
s.Now = func() time.Time { return now }
l, _ := s.CreateList("dev", "example.org", "Dev")
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte("m"), items(1)); err != nil {
t.Fatal(err)
}
now = now.Add(time.Hour)
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte("m"), items(1)); !errors.Is(err, ErrDuplicate) {
t.Fatalf("exactly one hour later it is still a duplicate, got %v", err)
}
now = now.Add(time.Second)
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte("m"), items(1)); err != nil {
t.Fatalf("after an hour the post is accepted again: %v", err)
}
if st, _ := s.OutboxStats(); st.Pending != 2 {
t.Fatalf("pending = %d", st.Pending)
}
}
func TestEnqueuePurgesExpiredRecordsOfEveryList(t *testing.T) {
s := newTestStore(t)
now := time.Unix(1_000_000, 0)
s.Now = func() time.Time { return now }
dev, _ := s.CreateList("dev", "example.org", "Dev")
ops, _ := s.CreateList("ops", "example.org", "Ops")
s.EnqueuePost(dev.ID, "mid:<a@x>", []byte("m"), items(1))
now = now.Add(2 * time.Hour)
s.EnqueuePost(ops.ID, "mid:<b@x>", []byte("m"), items(1))
var n int
if err := s.db.QueryRow(`SELECT COUNT(*) FROM delivered`).Scan(&n); err != nil || n != 1 {
t.Fatalf("records = %d (%v), want only the fresh one", n, err)
}
}
func TestEnqueuePostRecordsAndQueuesTogether(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
if _, err := s.db.Exec(`CREATE TRIGGER boom BEFORE INSERT ON outbox BEGIN SELECT RAISE(ABORT, 'boom'); END`); err != nil {
t.Fatal(err)
}
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte("m"), items(2)); err == nil {
t.Fatal("want the queue error")
}
s.db.Exec(`DROP TRIGGER boom`)
// The failed attempt left no record behind: the MTA's retry is queued.
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte("m"), items(2)); err != nil {
t.Fatalf("retry after a failed queue write: %v", err)
}
if st, _ := s.OutboxStats(); st.Pending != 2 {
t.Fatalf("pending = %d", st.Pending)
}
}
func TestEnqueuePostRecordsAPostWithNoRecipients(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte("m"), nil); err != nil {
t.Fatal(err)
}
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte("m"), nil); !errors.Is(err, ErrDuplicate) {
t.Fatalf("want ErrDuplicate, got %v", err)
}
}
func TestReleasingAHeldPostDeliversItEvenIfSeenAndRecordsIt(t *testing.T) {
s := newTestStore(t)
now := time.Unix(1_000_000, 0)
s.Now = func() time.Time { return now }
l, _ := s.CreateList("dev", "example.org", "Dev")
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte("m"), items(1)); err != nil {
t.Fatal(err)
}
hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r")
now = now.Add(50 * time.Minute)
if err := s.EnqueueFromHeld(hid, "mid:<a@x>", []byte("final"), items(2)); err != nil {
t.Fatalf("an approval always delivers: %v", err)
}
if st, _ := s.OutboxStats(); st.Pending != 3 {
t.Fatalf("pending = %d", st.Pending)
}
// The approval refreshed the record: a retry of the same post 20 minutes later is a duplicate.
now = now.Add(20 * time.Minute)
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte("m"), items(1)); !errors.Is(err, ErrDuplicate) {
t.Fatalf("want ErrDuplicate after a release, got %v", err)
}
}
func TestDeletingAListForgetsItsPosts(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
s.EnqueuePost(l.ID, "mid:<a@x>", []byte("m"), nil)
if err := s.DeleteList(l.ID); err != nil {
t.Fatal(err)
}
var n int
if err := s.db.QueryRow(`SELECT COUNT(*) FROM delivered`).Scan(&n); err != nil || n != 0 {
t.Fatalf("records = %d (%v)", n, err)
}
}

View file

@ -58,19 +58,61 @@ func (s *Store) Enqueue(msg []byte, items []NewOutbound) error {
return tx.Commit()
}
// EnqueueFromHeld releases a held message: it removes it from the moderation queue and queues
// its deliveries in one transaction. Of several simultaneous approvals only one succeeds, the
// others get ErrNotFound, so a message is never delivered twice. With no recipients the message
// is still released.
func (s *Store) EnqueueFromHeld(heldID int64, msg []byte, items []NewOutbound) error {
// postMemory is how long a post queued for a list is remembered, to refuse it if the MTA delivers it again.
const postMemory = time.Hour
// EnqueuePost queues a post to a list, like Enqueue, and records postID (the identity of the post)
// in the same transaction, so a crash never leaves one without the other. If the list already
// queued the same postID within postMemory it queues nothing and returns ErrDuplicate: the MTA
// delivers the post again when our reply to the first delivery was lost.
func (s *Store) EnqueuePost(listID int64, postID string, msg []byte, items []NewOutbound) error {
now := s.Now()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback()
if _, err := tx.Exec(`DELETE FROM delivered WHERE created_at<?`, now.Add(-postMemory).Unix()); err != nil {
return err
}
if _, err := tx.Exec(`INSERT INTO delivered(list_id, post_id, created_at) VALUES(?,?,?)`, listID, postID, now.Unix()); err != nil {
if isUnique(err) {
return ErrDuplicate
}
return err
}
if len(items) > 0 {
if err := enqueueTx(tx, now.Unix(), msg, items); err != nil {
return err
}
}
return tx.Commit()
}
// EnqueueFromHeld releases a held message: it removes it from the moderation queue and queues
// its deliveries in one transaction. Of several simultaneous approvals only one succeeds, the
// others get ErrNotFound, so a message is never delivered twice. With no recipients the message
// is still released. An approval always delivers, even a post already seen, and records postID
// as EnqueuePost does, so a later copy of the same post is refused.
func (s *Store) EnqueueFromHeld(heldID int64, postID string, msg []byte, items []NewOutbound) error {
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback()
var listID int64
if err := tx.QueryRow(`SELECT list_id FROM held WHERE id=?`, heldID).Scan(&listID); errors.Is(err, sql.ErrNoRows) {
return ErrNotFound
} else if err != nil {
return err
}
if err := affected(tx.Exec(`DELETE FROM held WHERE id=?`, heldID)); err != nil {
return err
}
if _, err := tx.Exec(`INSERT INTO delivered(list_id, post_id, created_at) VALUES(?,?,?)
ON CONFLICT(list_id, post_id) DO UPDATE SET created_at=excluded.created_at`, listID, postID, s.Now().Unix()); err != nil {
return err
}
if len(items) > 0 {
if err := enqueueTx(tx, s.Now().Unix(), msg, items); err != nil {
return err

View file

@ -194,7 +194,7 @@ func TestEnqueueFromHeldMovesTheMessageAtomically(t *testing.T) {
l, _ := s.CreateList("dev", "example.org", "Dev")
hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("Subject: s\r\n\r\nb"), "r")
if err := s.EnqueueFromHeld(hid, []byte("final"), items(2)); err != nil {
if err := s.EnqueueFromHeld(hid, "mid:<h@x>", []byte("final"), items(2)); err != nil {
t.Fatal(err)
}
if _, err := s.GetHeld(hid); !errors.Is(err, ErrNotFound) {
@ -204,7 +204,7 @@ func TestEnqueueFromHeldMovesTheMessageAtomically(t *testing.T) {
t.Fatalf("pending = %d", st.Pending)
}
// Approving again must not queue anything.
if err := s.EnqueueFromHeld(hid, []byte("final"), items(2)); !errors.Is(err, ErrNotFound) {
if err := s.EnqueueFromHeld(hid, "mid:<h@x>", []byte("final"), items(2)); !errors.Is(err, ErrNotFound) {
t.Fatalf("want ErrNotFound, got %v", err)
}
if st, _ := s.OutboxStats(); st.Pending != 2 {
@ -216,7 +216,7 @@ func TestEnqueueFromHeldWithNoRecipientsStillReleasesTheMessage(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r")
if err := s.EnqueueFromHeld(hid, []byte("final"), nil); err != nil {
if err := s.EnqueueFromHeld(hid, "mid:<h@x>", []byte("final"), nil); err != nil {
t.Fatal(err)
}
if _, err := s.GetHeld(hid); !errors.Is(err, ErrNotFound) {
@ -237,7 +237,7 @@ func TestOnlyOneOfSeveralSimultaneousApprovalsWins(t *testing.T) {
for i := 0; i < n; i++ {
go func() {
<-start
results <- s.EnqueueFromHeld(hid, []byte("final"), items(3))
results <- s.EnqueueFromHeld(hid, "mid:<h@x>", []byte("final"), items(3))
}()
}
close(start)

View file

@ -26,6 +26,8 @@ var (
ErrHeldTooLarge = errors.New("message too large to hold for moderation")
// ErrExists is returned when creating something that already exists.
ErrExists = errors.New("already exists")
// ErrDuplicate is returned when the same post was already queued for the list within postMemory.
ErrDuplicate = errors.New("post already delivered")
)
// Policy values.
@ -208,6 +210,12 @@ CREATE TABLE IF NOT EXISTS outbox (
failed INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS outbox_due ON outbox(next_at, id);
CREATE TABLE IF NOT EXISTS delivered (
list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE,
post_id TEXT NOT NULL,
created_at INTEGER NOT NULL,
PRIMARY KEY (list_id, post_id)
);
CREATE TABLE IF NOT EXISTS settings (
key TEXT PRIMARY KEY,
value TEXT NOT NULL