From bd4258ebaca826b1df7e8bf41deee895a294a9ff Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 11:52:25 +0100 Subject: [PATCH] feat(router): queue a post to a list only once per hour When the reply to a delivered post is lost, the MTA sends the post again and every member got it twice. The list now remembers the posts it queued. - store: new delivered table (list id, post id, time), created with CREATE TABLE IF NOT EXISTS and dropped with its list - store: EnqueuePost records the post and queues it in one transaction, returns ErrDuplicate when the list queued it in the last hour, and purges older records - store: EnqueueFromHeld records the post too, without refusing it: an approval always delivers - router: Deliver uses EnqueuePost and logs a duplicate; the post id is the Message-ID, or a SHA-256 of the message when it has none - router: Release passes the post id to EnqueueFromHeld - tests for duplicates, other ids and lists, expiry on the store clock, atomicity, posts without Message-ID and approvals --- internal/router/dup_test.go | 108 +++++++++++++++++++++++++ internal/router/router.go | 34 +++++++- internal/store/dup_test.go | 145 ++++++++++++++++++++++++++++++++++ internal/store/outbox.go | 52 ++++++++++-- internal/store/outbox_test.go | 8 +- internal/store/store.go | 8 ++ 6 files changed, 343 insertions(+), 12 deletions(-) create mode 100644 internal/router/dup_test.go create mode 100644 internal/store/dup_test.go diff --git a/internal/router/dup_test.go b/internal/router/dup_test.go new file mode 100644 index 0000000..829eee6 --- /dev/null +++ b/internal/router/dup_test.go @@ -0,0 +1,108 @@ +package router + +import ( + "errors" + "testing" +) + +func pending(t *testing.T, e *env) int { + t.Helper() + st, err := e.s.OutboxStats() + if err != nil { + t.Fatal(err) + } + return st.Pending +} + +func (e *env) post(t *testing.T, msg string) error { + t.Helper() + res, err := e.r.Route("alice@example.com", "dev@example.org", []byte(msg)) + if err != nil { + t.Fatalf("route: %v", err) + } + return e.r.Deliver(res) +} + +func TestTheSamePostIsDeliveredOnce(t *testing.T) { + e := newEnv(t) + woke := 0 + e.r.OnEnqueue = func() { woke++ } + msg := "From: alice@example.com\r\nSubject: hi\r\nMessage-ID: \r\n\r\nbody\r\n" + if err := e.post(t, msg); err != nil { + t.Fatal(err) + } + if err := e.post(t, msg); !errors.Is(err, ErrDuplicate) { + t.Fatalf("want ErrDuplicate, got %v", err) + } + if n := pending(t, e); n != 2 { + t.Fatalf("pending = %d, want one delivery per member", n) + } + if woke != 1 { + t.Fatalf("OnEnqueue called %d times for a duplicate", woke) + } +} + +func TestAnotherMessageIDIsAnotherPost(t *testing.T) { + e := newEnv(t) + if err := e.post(t, "From: alice@example.com\r\nMessage-ID: \r\n\r\nbody\r\n"); err != nil { + t.Fatal(err) + } + if err := e.post(t, "From: alice@example.com\r\nMessage-ID: \r\n\r\nbody\r\n"); err != nil { + t.Fatal(err) + } + if n := pending(t, e); n != 4 { + t.Fatalf("pending = %d", n) + } +} + +func TestAPostWithoutMessageIDIsKnownByItsContent(t *testing.T) { + e := newEnv(t) + msg := "From: alice@example.com\r\nSubject: hi\r\n\r\nbody\r\n" + if err := e.post(t, msg); err != nil { + t.Fatal(err) + } + if err := e.post(t, msg); !errors.Is(err, ErrDuplicate) { + t.Fatalf("the same bytes again: want ErrDuplicate, got %v", err) + } + if err := e.post(t, msg+"more\r\n"); err != nil { + t.Fatalf("another body is another post: %v", err) + } + if n := pending(t, e); n != 4 { + t.Fatalf("pending = %d", n) + } +} + +func TestPostIDIgnoresTheRestWhenThereIsAMessageID(t *testing.T) { + a := postID([]byte("Message-ID: \r\nSubject: a\r\n\r\nx")) + b := postID([]byte("Subject: b\r\nMessage-ID: \r\n\r\ny")) + if a != b || a != "mid:" { + t.Fatalf("%q %q", a, b) + } + if c := postID([]byte("Subject: b\r\n\r\ny")); c == postID([]byte("Subject: b\r\n\r\nz")) || len(c) < 10 { + t.Fatalf("content hash: %q", c) + } + if postID([]byte("Message-ID: \r\n\r\n")) == postID([]byte("Message-ID: \r\n\r\n")) { + t.Fatal("Message-IDs are case sensitive") + } +} + +func TestAnApprovedPostIsDeliveredEvenIfItWasSeen(t *testing.T) { + e := newEnv(t) + msg := "From: alice@example.com\r\nSubject: hi\r\nMessage-ID: \r\n\r\nbody\r\n" + if err := e.post(t, msg); err != nil { + t.Fatal(err) + } + id, err := e.s.HoldMessage(e.list.ID, "alice@example.com", []byte(msg), "r") + if err != nil { + t.Fatal(err) + } + if err := e.r.Release(id); err != nil { + t.Fatalf("release: %v", err) + } + if n := pending(t, e); n != 4 { + t.Fatalf("pending = %d, the approval must deliver", n) + } + if err := e.post(t, msg); !errors.Is(err, ErrDuplicate) { + t.Fatalf("a copy after the approval: want ErrDuplicate, got %v", err) + } +} diff --git a/internal/router/router.go b/internal/router/router.go index 3f9c23b..f1d57dd 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -3,6 +3,8 @@ package router import ( "bytes" + "crypto/sha256" + "encoding/hex" "errors" "log" "net/mail" @@ -34,6 +36,9 @@ var ( ErrGone = errors.New("already unsubscribed") // ErrJoinClosed means the list does not accept self-service subscriptions. ErrJoinClosed = errors.New("list is closed") + // ErrDuplicate means the list already queued the same post less than an hour ago: the MTA sent + // it again because our reply was lost. It is acknowledged and not queued again. + ErrDuplicate = store.ErrDuplicate ) // Recipient is a subscriber that receives a message. @@ -126,13 +131,36 @@ func (r *Router) prepare(l store.List, data []byte) (Result, error) { // Deliver queues the message for every recipient, each with its own signed // bounce address (VERP) and unsubscribe headers. The queue write is atomic: on -// error nothing was queued and the caller may safely retry. +// error nothing was queued and the caller may safely retry. A post the list +// already queued in the last hour is not queued again: Deliver returns ErrDuplicate. func (r *Router) Deliver(res Result) error { items, err := r.outbound(res) if err != nil { return err } - return r.enqueue(res.Data, items) + id := postID(res.Data) + if err := r.store.EnqueuePost(res.List.ID, id, res.Data, items); err != nil { + if errors.Is(err, ErrDuplicate) { + log.Printf("duplicate post to %s not delivered again: %s", res.List.Address(), id) + } + return err + } + if r.OnEnqueue != nil && len(items) > 0 { + r.OnEnqueue() + } + return nil +} + +// postID identifies a post for the duplicate check: its Message-ID, or a hash of the whole +// message when it has none. +func postID(data []byte) string { + if m, err := mail.ReadMessage(bytes.NewReader(data)); err == nil { + if id := strings.TrimSpace(m.Header.Get("Message-ID")); id != "" { + return "mid:" + id + } + } + sum := sha256.Sum256(data) + return "sha256:" + hex.EncodeToString(sum[:]) } // outbound builds one queue entry per recipient. @@ -213,7 +241,7 @@ func (r *Router) Release(heldID int64) error { if err != nil { return err } - if err := r.store.EnqueueFromHeld(heldID, res.Data, items); err != nil { + if err := r.store.EnqueueFromHeld(heldID, postID(res.Data), res.Data, items); err != nil { return err } if r.OnEnqueue != nil && len(items) > 0 { diff --git a/internal/store/dup_test.go b/internal/store/dup_test.go new file mode 100644 index 0000000..7c65861 --- /dev/null +++ b/internal/store/dup_test.go @@ -0,0 +1,145 @@ +package store + +import ( + "errors" + "testing" + "time" +) + +func TestEnqueuePostRefusesTheSamePostTwiceWithinAnHour(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + l, _ := s.CreateList("dev", "example.org", "Dev") + + if err := s.EnqueuePost(l.ID, "mid:", []byte("m"), items(2)); err != nil { + t.Fatal(err) + } + now = now.Add(59 * time.Minute) + if err := s.EnqueuePost(l.ID, "mid:", []byte("m"), items(2)); !errors.Is(err, ErrDuplicate) { + t.Fatalf("want ErrDuplicate, got %v", err) + } + if st, _ := s.OutboxStats(); st.Pending != 2 { + t.Fatalf("a duplicate must queue nothing, pending = %d", st.Pending) + } +} + +func TestEnqueuePostTellsPostsAndListsApart(t *testing.T) { + s := newTestStore(t) + dev, _ := s.CreateList("dev", "example.org", "Dev") + ops, _ := s.CreateList("ops", "example.org", "Ops") + if err := s.EnqueuePost(dev.ID, "mid:", []byte("m"), items(1)); err != nil { + t.Fatal(err) + } + if err := s.EnqueuePost(dev.ID, "mid:", []byte("m"), items(1)); err != nil { + t.Fatalf("another Message-ID: %v", err) + } + if err := s.EnqueuePost(ops.ID, "mid:", []byte("m"), items(1)); err != nil { + t.Fatalf("the same post to another list: %v", err) + } + if st, _ := s.OutboxStats(); st.Pending != 3 { + t.Fatalf("pending = %d", st.Pending) + } +} + +func TestEnqueuePostForgetsAPostAfterAnHour(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + l, _ := s.CreateList("dev", "example.org", "Dev") + if err := s.EnqueuePost(l.ID, "mid:", []byte("m"), items(1)); err != nil { + t.Fatal(err) + } + now = now.Add(time.Hour) + if err := s.EnqueuePost(l.ID, "mid:", []byte("m"), items(1)); !errors.Is(err, ErrDuplicate) { + t.Fatalf("exactly one hour later it is still a duplicate, got %v", err) + } + now = now.Add(time.Second) + if err := s.EnqueuePost(l.ID, "mid:", []byte("m"), items(1)); err != nil { + t.Fatalf("after an hour the post is accepted again: %v", err) + } + if st, _ := s.OutboxStats(); st.Pending != 2 { + t.Fatalf("pending = %d", st.Pending) + } +} + +func TestEnqueuePurgesExpiredRecordsOfEveryList(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + dev, _ := s.CreateList("dev", "example.org", "Dev") + ops, _ := s.CreateList("ops", "example.org", "Ops") + s.EnqueuePost(dev.ID, "mid:", []byte("m"), items(1)) + now = now.Add(2 * time.Hour) + s.EnqueuePost(ops.ID, "mid:", []byte("m"), items(1)) + var n int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM delivered`).Scan(&n); err != nil || n != 1 { + t.Fatalf("records = %d (%v), want only the fresh one", n, err) + } +} + +func TestEnqueuePostRecordsAndQueuesTogether(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + if _, err := s.db.Exec(`CREATE TRIGGER boom BEFORE INSERT ON outbox BEGIN SELECT RAISE(ABORT, 'boom'); END`); err != nil { + t.Fatal(err) + } + if err := s.EnqueuePost(l.ID, "mid:", []byte("m"), items(2)); err == nil { + t.Fatal("want the queue error") + } + s.db.Exec(`DROP TRIGGER boom`) + // The failed attempt left no record behind: the MTA's retry is queued. + if err := s.EnqueuePost(l.ID, "mid:", []byte("m"), items(2)); err != nil { + t.Fatalf("retry after a failed queue write: %v", err) + } + if st, _ := s.OutboxStats(); st.Pending != 2 { + t.Fatalf("pending = %d", st.Pending) + } +} + +func TestEnqueuePostRecordsAPostWithNoRecipients(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + if err := s.EnqueuePost(l.ID, "mid:", []byte("m"), nil); err != nil { + t.Fatal(err) + } + if err := s.EnqueuePost(l.ID, "mid:", []byte("m"), nil); !errors.Is(err, ErrDuplicate) { + t.Fatalf("want ErrDuplicate, got %v", err) + } +} + +func TestReleasingAHeldPostDeliversItEvenIfSeenAndRecordsIt(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + l, _ := s.CreateList("dev", "example.org", "Dev") + if err := s.EnqueuePost(l.ID, "mid:", []byte("m"), items(1)); err != nil { + t.Fatal(err) + } + hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r") + now = now.Add(50 * time.Minute) + if err := s.EnqueueFromHeld(hid, "mid:", []byte("final"), items(2)); err != nil { + t.Fatalf("an approval always delivers: %v", err) + } + if st, _ := s.OutboxStats(); st.Pending != 3 { + t.Fatalf("pending = %d", st.Pending) + } + // The approval refreshed the record: a retry of the same post 20 minutes later is a duplicate. + now = now.Add(20 * time.Minute) + if err := s.EnqueuePost(l.ID, "mid:", []byte("m"), items(1)); !errors.Is(err, ErrDuplicate) { + t.Fatalf("want ErrDuplicate after a release, got %v", err) + } +} + +func TestDeletingAListForgetsItsPosts(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.EnqueuePost(l.ID, "mid:", []byte("m"), nil) + if err := s.DeleteList(l.ID); err != nil { + t.Fatal(err) + } + var n int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM delivered`).Scan(&n); err != nil || n != 0 { + t.Fatalf("records = %d (%v)", n, err) + } +} diff --git a/internal/store/outbox.go b/internal/store/outbox.go index aaa3e20..c72abea 100644 --- a/internal/store/outbox.go +++ b/internal/store/outbox.go @@ -58,19 +58,61 @@ func (s *Store) Enqueue(msg []byte, items []NewOutbound) error { return tx.Commit() } -// EnqueueFromHeld releases a held message: it removes it from the moderation queue and queues -// its deliveries in one transaction. Of several simultaneous approvals only one succeeds, the -// others get ErrNotFound, so a message is never delivered twice. With no recipients the message -// is still released. -func (s *Store) EnqueueFromHeld(heldID int64, msg []byte, items []NewOutbound) error { +// postMemory is how long a post queued for a list is remembered, to refuse it if the MTA delivers it again. +const postMemory = time.Hour + +// EnqueuePost queues a post to a list, like Enqueue, and records postID (the identity of the post) +// in the same transaction, so a crash never leaves one without the other. If the list already +// queued the same postID within postMemory it queues nothing and returns ErrDuplicate: the MTA +// delivers the post again when our reply to the first delivery was lost. +func (s *Store) EnqueuePost(listID int64, postID string, msg []byte, items []NewOutbound) error { + now := s.Now() tx, err := s.db.Begin() if err != nil { return err } defer tx.Rollback() + if _, err := tx.Exec(`DELETE FROM delivered WHERE created_at 0 { + if err := enqueueTx(tx, now.Unix(), msg, items); err != nil { + return err + } + } + return tx.Commit() +} + +// EnqueueFromHeld releases a held message: it removes it from the moderation queue and queues +// its deliveries in one transaction. Of several simultaneous approvals only one succeeds, the +// others get ErrNotFound, so a message is never delivered twice. With no recipients the message +// is still released. An approval always delivers, even a post already seen, and records postID +// as EnqueuePost does, so a later copy of the same post is refused. +func (s *Store) EnqueueFromHeld(heldID int64, postID string, msg []byte, items []NewOutbound) error { + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + var listID int64 + if err := tx.QueryRow(`SELECT list_id FROM held WHERE id=?`, heldID).Scan(&listID); errors.Is(err, sql.ErrNoRows) { + return ErrNotFound + } else if err != nil { + return err + } if err := affected(tx.Exec(`DELETE FROM held WHERE id=?`, heldID)); err != nil { return err } + if _, err := tx.Exec(`INSERT INTO delivered(list_id, post_id, created_at) VALUES(?,?,?) + ON CONFLICT(list_id, post_id) DO UPDATE SET created_at=excluded.created_at`, listID, postID, s.Now().Unix()); err != nil { + return err + } if len(items) > 0 { if err := enqueueTx(tx, s.Now().Unix(), msg, items); err != nil { return err diff --git a/internal/store/outbox_test.go b/internal/store/outbox_test.go index 70d95a9..3458932 100644 --- a/internal/store/outbox_test.go +++ b/internal/store/outbox_test.go @@ -194,7 +194,7 @@ func TestEnqueueFromHeldMovesTheMessageAtomically(t *testing.T) { l, _ := s.CreateList("dev", "example.org", "Dev") hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("Subject: s\r\n\r\nb"), "r") - if err := s.EnqueueFromHeld(hid, []byte("final"), items(2)); err != nil { + if err := s.EnqueueFromHeld(hid, "mid:", []byte("final"), items(2)); err != nil { t.Fatal(err) } if _, err := s.GetHeld(hid); !errors.Is(err, ErrNotFound) { @@ -204,7 +204,7 @@ func TestEnqueueFromHeldMovesTheMessageAtomically(t *testing.T) { t.Fatalf("pending = %d", st.Pending) } // Approving again must not queue anything. - if err := s.EnqueueFromHeld(hid, []byte("final"), items(2)); !errors.Is(err, ErrNotFound) { + if err := s.EnqueueFromHeld(hid, "mid:", []byte("final"), items(2)); !errors.Is(err, ErrNotFound) { t.Fatalf("want ErrNotFound, got %v", err) } if st, _ := s.OutboxStats(); st.Pending != 2 { @@ -216,7 +216,7 @@ func TestEnqueueFromHeldWithNoRecipientsStillReleasesTheMessage(t *testing.T) { s := newTestStore(t) l, _ := s.CreateList("dev", "example.org", "Dev") hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r") - if err := s.EnqueueFromHeld(hid, []byte("final"), nil); err != nil { + if err := s.EnqueueFromHeld(hid, "mid:", []byte("final"), nil); err != nil { t.Fatal(err) } if _, err := s.GetHeld(hid); !errors.Is(err, ErrNotFound) { @@ -237,7 +237,7 @@ func TestOnlyOneOfSeveralSimultaneousApprovalsWins(t *testing.T) { for i := 0; i < n; i++ { go func() { <-start - results <- s.EnqueueFromHeld(hid, []byte("final"), items(3)) + results <- s.EnqueueFromHeld(hid, "mid:", []byte("final"), items(3)) }() } close(start) diff --git a/internal/store/store.go b/internal/store/store.go index 5abaedf..8d31779 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -26,6 +26,8 @@ var ( ErrHeldTooLarge = errors.New("message too large to hold for moderation") // ErrExists is returned when creating something that already exists. ErrExists = errors.New("already exists") + // ErrDuplicate is returned when the same post was already queued for the list within postMemory. + ErrDuplicate = errors.New("post already delivered") ) // Policy values. @@ -208,6 +210,12 @@ CREATE TABLE IF NOT EXISTS outbox ( failed INTEGER NOT NULL DEFAULT 0 ); CREATE INDEX IF NOT EXISTS outbox_due ON outbox(next_at, id); +CREATE TABLE IF NOT EXISTS delivered ( + list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE, + post_id TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (list_id, post_id) +); CREATE TABLE IF NOT EXISTS settings ( key TEXT PRIMARY KEY, value TEXT NOT NULL