parsimony/internal/store/outbox_test.go
tgiachi bd4258ebac feat(router): queue a post to a list only once per hour
When the reply to a delivered post is lost, the MTA sends the post again
and every member got it twice. The list now remembers the posts it queued.

- store: new delivered table (list id, post id, time), created with
  CREATE TABLE IF NOT EXISTS and dropped with its list
- store: EnqueuePost records the post and queues it in one transaction,
  returns ErrDuplicate when the list queued it in the last hour, and
  purges older records
- store: EnqueueFromHeld records the post too, without refusing it: an
  approval always delivers
- router: Deliver uses EnqueuePost and logs a duplicate; the post id is
  the Message-ID, or a SHA-256 of the message when it has none
- router: Release passes the post id to EnqueueFromHeld
- tests for duplicates, other ids and lists, expiry on the store clock,
  atomicity, posts without Message-ID and approvals
2026-11-10 11:52:25 +01:00

370 lines
11 KiB
Go

package store
import (
"errors"
"testing"
"time"
)
func items(n int) []NewOutbound {
out := make([]NewOutbound, n)
for i := range out {
out[i] = NewOutbound{From: "bounce@x.org", Rcpt: string(rune('a'+i)) + "@x.com", MemberID: int64(i + 1), Prefix: []byte("List-Unsubscribe: <u>\r\n")}
}
return out
}
func TestEnqueueAndDue(t *testing.T) {
s := newTestStore(t)
now := time.Unix(1_000_000, 0)
s.Now = func() time.Time { return now }
if err := s.Enqueue([]byte("Subject: x\r\n\r\nbody"), items(3)); err != nil {
t.Fatal(err)
}
due, err := s.DueOutbox(10)
if err != nil || len(due) != 3 {
t.Fatalf("due: %d %v", len(due), err)
}
d := due[0]
if d.From != "bounce@x.org" || d.Rcpt != "a@x.com" || d.MemberID != 1 || string(d.Prefix) != "List-Unsubscribe: <u>\r\n" || d.Attempts != 0 {
t.Fatalf("item: %+v", d)
}
data, err := s.OutboxData(d.MsgID)
if err != nil || string(data) != "Subject: x\r\n\r\nbody" {
t.Fatalf("data: %q %v", data, err)
}
if due[0].MsgID != due[2].MsgID {
t.Fatal("recipients of one post must share the stored body")
}
if got, _ := s.DueOutbox(2); len(got) != 2 {
t.Fatalf("limit not applied: %d", len(got))
}
}
func TestEnqueueNothingStoresNothing(t *testing.T) {
s := newTestStore(t)
if err := s.Enqueue([]byte("x"), nil); err != nil {
t.Fatal(err)
}
var n int
s.db.QueryRow(`SELECT COUNT(*) FROM outbox_msgs`).Scan(&n)
if n != 0 {
t.Fatalf("orphan message stored: %d", n)
}
}
func TestCompleteDeletesTheBodyWithTheLastRecipient(t *testing.T) {
s := newTestStore(t)
s.Enqueue([]byte("body"), items(2))
due, _ := s.DueOutbox(10)
if err := s.CompleteOutbox(due[0].ID); err != nil {
t.Fatal(err)
}
if _, err := s.OutboxData(due[0].MsgID); err != nil {
t.Fatalf("body must stay while another recipient waits: %v", err)
}
if err := s.CompleteOutbox(due[1].ID); err != nil {
t.Fatal(err)
}
if _, err := s.OutboxData(due[0].MsgID); !errors.Is(err, ErrNotFound) {
t.Fatalf("body must go with the last recipient: %v", err)
}
if err := s.CompleteOutbox(due[1].ID); !errors.Is(err, ErrNotFound) {
t.Fatalf("want ErrNotFound, got %v", err)
}
}
func TestRetryDelaysAndCounts(t *testing.T) {
s := newTestStore(t)
now := time.Unix(1_000_000, 0)
s.Now = func() time.Time { return now }
s.Enqueue([]byte("body"), items(2))
due, _ := s.DueOutbox(10)
next := now.Add(5 * time.Minute)
if err := s.RetryOutbox(due[0].ID, next, "connection refused"); err != nil {
t.Fatal(err)
}
got, _ := s.DueOutbox(10)
if len(got) != 1 || got[0].ID != due[1].ID {
t.Fatalf("the retried item must wait: %+v", got)
}
now = next.Add(-time.Second)
if got, _ := s.DueOutbox(10); len(got) != 1 {
t.Fatal("not due one second early")
}
now = next
got, _ = s.DueOutbox(10)
if len(got) != 2 {
t.Fatalf("due again at next_at: %d", len(got))
}
var retried OutboxItem
for _, g := range got {
if g.ID == due[0].ID {
retried = g
}
}
if retried.Attempts != 1 || retried.LastError != "connection refused" || !retried.NextAt.Equal(next) {
t.Fatalf("retried: %+v", retried)
}
if err := s.RetryOutbox(999, next, "x"); !errors.Is(err, ErrNotFound) {
t.Fatalf("want ErrNotFound, got %v", err)
}
}
func TestDueIsOrderedByNextAttempt(t *testing.T) {
s := newTestStore(t)
now := time.Unix(1_000_000, 0)
s.Now = func() time.Time { return now }
s.Enqueue([]byte("body"), items(3))
due, _ := s.DueOutbox(10)
s.RetryOutbox(due[2].ID, now.Add(-time.Hour), "e") // the LAST id, but the longest waiting
now = now.Add(time.Minute)
got, _ := s.DueOutbox(10)
if got[0].ID != due[2].ID {
t.Fatalf("the longest waiting item goes first: %+v", got)
}
}
func TestOutboxStats(t *testing.T) {
s := newTestStore(t)
now := time.Unix(1_000_000, 0)
s.Now = func() time.Time { return now }
st, err := s.OutboxStats()
if err != nil || st.Pending != 0 || st.Retrying != 0 || !st.Oldest.IsZero() {
t.Fatalf("empty: %+v %v", st, err)
}
s.Enqueue([]byte("body"), items(3))
due, _ := s.DueOutbox(10)
s.RetryOutbox(due[0].ID, now.Add(time.Minute), "boom")
now = now.Add(10 * time.Second)
st, _ = s.OutboxStats()
if st.Pending != 3 || st.Retrying != 1 || !st.Oldest.Equal(time.Unix(1_000_000, 0)) {
t.Fatalf("stats: %+v", st)
}
}
func TestListOutbox(t *testing.T) {
s := newTestStore(t)
s.Enqueue([]byte("body"), items(5))
due, _ := s.DueOutbox(10)
s.RetryOutbox(due[3].ID, time.Now().Add(time.Hour), "refused")
got, err := s.ListOutbox(3)
if err != nil || len(got) != 3 {
t.Fatalf("list: %d %v", len(got), err)
}
failing, _ := s.ListOutbox(10)
if failing[0].ID != due[3].ID {
t.Fatalf("items that failed come first: %+v", failing[0])
}
}
func TestDiscardOutbox(t *testing.T) {
s := newTestStore(t)
s.Enqueue([]byte("body"), items(1))
due, _ := s.DueOutbox(10)
if err := s.CompleteOutbox(due[0].ID); err != nil {
t.Fatal(err)
}
if err := s.CompleteOutbox(due[0].ID); !errors.Is(err, ErrNotFound) {
t.Fatalf("want ErrNotFound, got %v", err)
}
}
func TestQueueSurvivesReopen(t *testing.T) {
path := t.TempDir() + "/q.db"
s, _ := Open(path)
s.Enqueue([]byte("persist me"), items(2))
s.Close()
s2, err := Open(path)
if err != nil {
t.Fatal(err)
}
defer s2.Close()
due, _ := s2.DueOutbox(10)
if len(due) != 2 {
t.Fatalf("queue lost on restart: %d", len(due))
}
if data, _ := s2.OutboxData(due[0].MsgID); string(data) != "persist me" {
t.Fatalf("data: %q", data)
}
}
func TestEnqueueFromHeldMovesTheMessageAtomically(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("Subject: s\r\n\r\nb"), "r")
if err := s.EnqueueFromHeld(hid, "mid:<h@x>", []byte("final"), items(2)); err != nil {
t.Fatal(err)
}
if _, err := s.GetHeld(hid); !errors.Is(err, ErrNotFound) {
t.Fatal("the held message is gone")
}
if st, _ := s.OutboxStats(); st.Pending != 2 {
t.Fatalf("pending = %d", st.Pending)
}
// Approving again must not queue anything.
if err := s.EnqueueFromHeld(hid, "mid:<h@x>", []byte("final"), items(2)); !errors.Is(err, ErrNotFound) {
t.Fatalf("want ErrNotFound, got %v", err)
}
if st, _ := s.OutboxStats(); st.Pending != 2 {
t.Fatalf("a second approval queued more mail: %d", st.Pending)
}
}
func TestEnqueueFromHeldWithNoRecipientsStillReleasesTheMessage(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r")
if err := s.EnqueueFromHeld(hid, "mid:<h@x>", []byte("final"), nil); err != nil {
t.Fatal(err)
}
if _, err := s.GetHeld(hid); !errors.Is(err, ErrNotFound) {
t.Fatal("the message is released even if nobody is subscribed")
}
if st, _ := s.OutboxStats(); st.Pending != 0 {
t.Fatalf("pending = %d", st.Pending)
}
}
func TestOnlyOneOfSeveralSimultaneousApprovalsWins(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r")
const n = 12
results := make(chan error, n)
start := make(chan struct{})
for i := 0; i < n; i++ {
go func() {
<-start
results <- s.EnqueueFromHeld(hid, "mid:<h@x>", []byte("final"), items(3))
}()
}
close(start)
wins, notFound := 0, 0
for i := 0; i < n; i++ {
switch err := <-results; {
case err == nil:
wins++
case errors.Is(err, ErrNotFound):
notFound++
default:
t.Fatalf("unexpected error: %v", err)
}
}
if wins != 1 || notFound != n-1 {
t.Fatalf("wins=%d notFound=%d", wins, notFound)
}
if st, _ := s.OutboxStats(); st.Pending != 3 {
t.Fatalf("the message must be queued exactly once for 3 recipients, pending = %d", st.Pending)
}
}
func TestFailedDeliveriesStayVisibleButAreNeverRetried(t *testing.T) {
s := newTestStore(t)
now := time.Unix(1_000_000, 0)
s.Now = func() time.Time { return now }
s.Enqueue([]byte("body"), items(3))
due, _ := s.DueOutbox(10)
if err := s.FailOutbox(due[0].ID, "relay said: 554 policy rejection"); err != nil {
t.Fatal(err)
}
if err := s.FailOutbox(999, "x"); !errors.Is(err, ErrNotFound) {
t.Fatalf("want ErrNotFound, got %v", err)
}
got, _ := s.DueOutbox(10)
if len(got) != 2 {
t.Fatalf("a failed delivery is not due again: %d due", len(got))
}
now = now.Add(24 * time.Hour)
if got, _ := s.DueOutbox(10); len(got) != 2 {
t.Fatal("not even a day later")
}
st, _ := s.OutboxStats()
if st.Pending != 2 || st.Failed != 1 || st.Retrying != 0 {
t.Fatalf("stats: %+v", st)
}
list, _ := s.ListOutbox(10)
if len(list) != 3 || !list[0].Failed || list[0].ID != due[0].ID || list[0].LastError != "relay said: 554 policy rejection" {
t.Fatalf("failed deliveries come first, with the reason: %+v", list)
}
// The admin can discard it, like any other.
if err := s.CompleteOutbox(due[0].ID); err != nil {
t.Fatal(err)
}
if st, _ := s.OutboxStats(); st.Failed != 0 {
t.Fatalf("%+v", st)
}
}
func TestFailedDeliveriesAreForgottenAfterAWeek(t *testing.T) {
s := newTestStore(t)
now := time.Unix(1_000_000, 0)
s.Now = func() time.Time { return now }
s.Enqueue([]byte("old body"), items(1))
s.Enqueue([]byte("new body"), items(1))
due, _ := s.DueOutbox(10)
s.FailOutbox(due[0].ID, "old failure")
now = now.Add(5 * 24 * time.Hour)
s.FailOutbox(due[1].ID, "recent failure")
now = now.Add(3 * 24 * time.Hour) // the first failure is now 8 days old, the second 3
n, err := s.PurgeFailedOutbox()
if err != nil || n != 1 {
t.Fatalf("purged %d: %v", n, err)
}
list, _ := s.ListOutbox(10)
if len(list) != 1 || list[0].LastError != "recent failure" {
t.Fatalf("list: %+v", list)
}
if _, err := s.OutboxData(due[0].MsgID); !errors.Is(err, ErrNotFound) {
t.Fatal("the stored message of a purged delivery goes too")
}
if _, err := s.OutboxData(due[1].MsgID); err != nil {
t.Fatalf("the recent one keeps its message: %v", err)
}
// Waiting and retrying deliveries are never purged, however old.
s.Enqueue([]byte("waiting"), items(1))
now = now.Add(365 * 24 * time.Hour)
s.PurgeFailedOutbox()
if st, _ := s.OutboxStats(); st.Pending != 1 {
t.Fatalf("a waiting delivery must survive: %+v", st)
}
}
func TestMigrationAddsTheFailedColumn(t *testing.T) {
path := t.TempDir() + "/old.db"
s, _ := Open(path)
s.Enqueue([]byte("x"), items(1))
s.db.Exec(`ALTER TABLE outbox DROP COLUMN failed`)
s.Close()
s2, err := Open(path)
if err != nil {
t.Fatalf("migrate: %v", err)
}
defer s2.Close()
due, _ := s2.DueOutbox(10)
if len(due) != 1 || due[0].Failed {
t.Fatalf("an existing delivery stays waiting: %+v", due)
}
}
func TestFailedComeBeforeDeliveriesThatRetriedMoreOften(t *testing.T) {
s := newTestStore(t)
s.Enqueue([]byte("body"), items(3))
due, _ := s.DueOutbox(10)
for i := 0; i < 4; i++ { // a delivery that has been retried 4 times
s.RetryOutbox(due[1].ID, time.Now().Add(time.Hour), "down")
}
s.RetryOutbox(due[2].ID, time.Now().Add(time.Hour), "down") // retried once
s.FailOutbox(due[0].ID, "refused") // failed, with fewer attempts than the others
list, _ := s.ListOutbox(10)
if list[0].ID != due[0].ID || !list[0].Failed {
t.Fatalf("a failed delivery is listed first whatever the others' attempts: %+v", list)
}
if list[1].ID != due[1].ID {
t.Fatalf("then the most retried: %+v", list)
}
}