parsimony/internal/store/posts_test.go
tgiachi 2abd435494 feat(store): record posts and count their deliveries
- posts table: one row per post queued to a list (message id, subject,
  time, recipients, sent, failed, dropped), deleted with the list and
  forgotten after 400 days when a new post is recorded
- EnqueuePost and EnqueueFromHeld record the post in the same
  transaction and stamp every delivery with its id; a refused duplicate
  records nothing; other mail keeps post_id 0
- CompleteOutbox counts sent, the new DropOutbox counts dropped and
  FailOutbox counts failed, each in the transaction that changes the
  delivery; a delivery that already failed is never counted twice
- outbox priority column; DueOutbox orders by priority, next_at, id
- ListPosts newest first (default 50, at most 200) with Pending derived
- additive migration of the outbox columns, tested on an old database
2026-11-10 12:37:52 +01:00

421 lines
13 KiB
Go

package store
import (
"errors"
"strings"
"testing"
"time"
)
const postMsg = "Message-ID: <a@x>\r\nSubject: =?utf-8?q?Caff=C3=A8?= news\x07\r\n\r\nbody"
func onlyPost(t *testing.T, s *Store, listID int64) Post {
t.Helper()
posts, err := s.ListPosts(listID, 0)
if err != nil || len(posts) != 1 {
t.Fatalf("posts: %+v %v", posts, err)
}
return posts[0]
}
func outboxPostIDs(t *testing.T, s *Store) []int64 {
t.Helper()
rows, err := s.db.Query(`SELECT post_id FROM outbox ORDER BY id`)
if err != nil {
t.Fatal(err)
}
defer rows.Close()
var ids []int64
for rows.Next() {
var id int64
rows.Scan(&id)
ids = append(ids, id)
}
return ids
}
func TestEnqueuePostRecordsThePost(t *testing.T) {
s := newTestStore(t)
now := time.Unix(1_000_000, 0)
s.Now = func() time.Time { return now }
l, _ := s.CreateList("dev", "example.org", "Dev")
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), items(3)); err != nil {
t.Fatal(err)
}
p := onlyPost(t, s, l.ID)
if p.ID == 0 || p.ListID != l.ID || p.MessageID != "<a@x>" || p.Subject != "Caffè news" || !p.CreatedAt.Equal(now) ||
p.Recipients != 3 || p.Sent != 0 || p.Failed != 0 || p.Dropped != 0 || p.Pending != 3 {
t.Fatalf("post: %+v", p)
}
for _, id := range outboxPostIDs(t, s) {
if id != p.ID {
t.Fatalf("every delivery carries the post id: %d != %d", id, p.ID)
}
}
}
func TestEnqueuePostRecordsAPostRowWithNoRecipients(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), nil); err != nil {
t.Fatal(err)
}
if p := onlyPost(t, s, l.ID); p.Recipients != 0 || p.Pending != 0 {
t.Fatalf("post: %+v", p)
}
}
func TestARefusedDuplicateRecordsNoPost(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), items(1))
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), items(1)); !errors.Is(err, ErrDuplicate) {
t.Fatalf("want ErrDuplicate, got %v", err)
}
onlyPost(t, s, l.ID)
}
func TestAFailedQueueWriteRecordsNoPost(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
if _, err := s.db.Exec(`CREATE TRIGGER boom BEFORE INSERT ON outbox BEGIN SELECT RAISE(ABORT, 'boom'); END`); err != nil {
t.Fatal(err)
}
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), items(2)); err == nil {
t.Fatal("want the queue error")
}
hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r")
if err := s.EnqueueFromHeld(hid, "mid:<h@x>", []byte(postMsg), items(2)); err == nil {
t.Fatal("want the queue error")
}
if posts, _ := s.ListPosts(l.ID, 0); len(posts) != 0 {
t.Fatalf("posts: %+v", posts)
}
}
func TestReleasingAHeldPostRecordsItOnce(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r")
if err := s.EnqueueFromHeld(hid, "mid:<a@x>", []byte(postMsg), items(2)); err != nil {
t.Fatal(err)
}
s.EnqueueFromHeld(hid, "mid:<a@x>", []byte(postMsg), items(2)) // a second approval: ErrNotFound
p := onlyPost(t, s, l.ID)
if p.Recipients != 2 || p.Subject != "Caffè news" {
t.Fatalf("post: %+v", p)
}
for _, id := range outboxPostIDs(t, s) {
if id != p.ID {
t.Fatalf("delivery post id %d != %d", id, p.ID)
}
}
hid2, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r")
if err := s.EnqueueFromHeld(hid2, "mid:<b@x>", []byte(postMsg), nil); err != nil {
t.Fatal(err)
}
if posts, _ := s.ListPosts(l.ID, 0); len(posts) != 2 || posts[0].Recipients != 0 {
t.Fatalf("a released post with no recipients has a row too: %+v", posts)
}
}
func TestOtherMailIsNotAPost(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
if err := s.Enqueue([]byte(postMsg), items(2)); err != nil {
t.Fatal(err)
}
if posts, _ := s.ListPosts(l.ID, 0); len(posts) != 0 {
t.Fatalf("posts: %+v", posts)
}
for _, id := range outboxPostIDs(t, s) {
if id != 0 {
t.Fatalf("post id %d", id)
}
}
}
func TestPostSubjectIsCutAndClean(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
msg := "Message-ID: <a\x1b@x>\r\nSubject: " + strings.Repeat("é", 300) + "\r\n\r\nb"
s.EnqueuePost(l.ID, "mid:x", []byte(msg), items(1))
p := onlyPost(t, s, l.ID)
if p.Subject != strings.Repeat("é", 200) || p.MessageID != "<a@x>" {
t.Fatalf("post: %+v", p)
}
}
func TestCountersFollowTheDeliveries(t *testing.T) {
s := newTestStore(t)
now := time.Unix(1_000_000, 0)
s.Now = func() time.Time { return now }
l, _ := s.CreateList("dev", "example.org", "Dev")
s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), items(6))
due, _ := s.DueOutbox(10)
if err := s.RetryOutbox(due[0].ID, now.Add(time.Minute), "down"); err != nil {
t.Fatal(err)
}
if p := onlyPost(t, s, l.ID); p.Sent+p.Failed+p.Dropped != 0 || p.Pending != 6 {
t.Fatalf("a retry changes nothing: %+v", p)
}
if err := s.CompleteOutbox(due[1].ID); err != nil {
t.Fatal(err)
}
if err := s.FailOutbox(due[2].ID, "refused"); err != nil {
t.Fatal(err)
}
if err := s.DropOutbox(due[3].ID); err != nil {
t.Fatal(err)
}
if err := s.DropOutbox(due[4].ID); err != nil {
t.Fatal(err)
}
p := onlyPost(t, s, l.ID)
if p.Sent != 1 || p.Failed != 1 || p.Dropped != 2 || p.Pending != 2 {
t.Fatalf("post: %+v", p)
}
// Discarding a failed delivery: it is already counted as failed.
if err := s.DropOutbox(due[2].ID); err != nil {
t.Fatal(err)
}
if err := s.CompleteOutbox(due[2].ID); !errors.Is(err, ErrNotFound) {
t.Fatalf("want ErrNotFound, got %v", err)
}
if err := s.DropOutbox(due[2].ID); !errors.Is(err, ErrNotFound) {
t.Fatalf("want ErrNotFound, got %v", err)
}
if p2 := onlyPost(t, s, l.ID); p2 != p {
t.Fatalf("discarding a failed delivery changes nothing: %+v", p2)
}
}
func TestCompletingAFailedDeliveryDoesNotCountItAsSent(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), items(1))
due, _ := s.DueOutbox(10)
s.FailOutbox(due[0].ID, "refused")
if err := s.CompleteOutbox(due[0].ID); err != nil {
t.Fatal(err)
}
if p := onlyPost(t, s, l.ID); p.Sent != 0 || p.Failed != 1 || p.Pending != 0 {
t.Fatalf("post: %+v", p)
}
}
func TestDropOutboxRemovesTheDeliveryAndTheBody(t *testing.T) {
s := newTestStore(t)
s.Enqueue([]byte("body"), items(1))
due, _ := s.DueOutbox(10)
if err := s.DropOutbox(due[0].ID); err != nil {
t.Fatal(err)
}
if st, _ := s.OutboxStats(); st.Pending != 0 || st.Failed != 0 {
t.Fatalf("stats: %+v", st)
}
if _, err := s.OutboxData(due[0].MsgID); !errors.Is(err, ErrNotFound) {
t.Fatalf("body: %v", err)
}
}
func TestPurgingOldFailuresKeepsTheCounters(t *testing.T) {
s := newTestStore(t)
now := time.Unix(1_000_000, 0)
s.Now = func() time.Time { return now }
l, _ := s.CreateList("dev", "example.org", "Dev")
s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), items(2))
due, _ := s.DueOutbox(10)
s.FailOutbox(due[0].ID, "refused")
now = now.Add(8 * 24 * time.Hour)
if n, err := s.PurgeFailedOutbox(); err != nil || n != 1 {
t.Fatalf("purged %d %v", n, err)
}
if p := onlyPost(t, s, l.ID); p.Failed != 1 || p.Pending != 1 {
t.Fatalf("post: %+v", p)
}
}
func TestListPostsNewestFirstWithALimit(t *testing.T) {
s := newTestStore(t)
now := time.Unix(1_000_000, 0)
s.Now = func() time.Time { return now }
dev, _ := s.CreateList("dev", "example.org", "Dev")
ops, _ := s.CreateList("ops", "example.org", "Ops")
for i := 0; i < 210; i++ {
now = now.Add(time.Second)
if err := s.EnqueuePost(dev.ID, "mid:"+string(rune(0x100+i)), []byte(postMsg), nil); err != nil {
t.Fatal(err)
}
}
s.EnqueuePost(ops.ID, "mid:ops", []byte(postMsg), items(1))
all, _ := s.ListPosts(dev.ID, 1000)
if len(all) != 200 {
t.Fatalf("capped at 200: %d", len(all))
}
for i := 1; i < len(all); i++ {
if !all[i-1].CreatedAt.After(all[i].CreatedAt) {
t.Fatalf("not newest first at %d", i)
}
}
if def, _ := s.ListPosts(dev.ID, 0); len(def) != 50 || def[0].ID != all[0].ID {
t.Fatalf("default 50: %d", len(def))
}
if neg, _ := s.ListPosts(dev.ID, -1); len(neg) != 50 {
t.Fatalf("default 50: %d", len(neg))
}
if some, _ := s.ListPosts(dev.ID, 7); len(some) != 7 {
t.Fatalf("limit 7: %d", len(some))
}
if exact, _ := s.ListPosts(dev.ID, 200); len(exact) != 200 {
t.Fatalf("limit 200: %d", len(exact))
}
if o, _ := s.ListPosts(ops.ID, 0); len(o) != 1 || o[0].Recipients != 1 {
t.Fatalf("other list: %+v", o)
}
}
func TestListPostsBreaksTiesByID(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
s.Now = func() time.Time { return time.Unix(1_000_000, 0) }
s.EnqueuePost(l.ID, "mid:1", []byte(postMsg), nil)
s.EnqueuePost(l.ID, "mid:2", []byte(postMsg), nil)
posts, _ := s.ListPosts(l.ID, 0)
if len(posts) != 2 || posts[0].ID < posts[1].ID {
t.Fatalf("the later post first: %+v", posts)
}
}
func TestOldPostsArePurgedWhenANewOneIsRecorded(t *testing.T) {
s := newTestStore(t)
now := time.Unix(100_000_000, 0)
s.Now = func() time.Time { return now }
dev, _ := s.CreateList("dev", "example.org", "Dev")
ops, _ := s.CreateList("ops", "example.org", "Ops")
s.EnqueuePost(dev.ID, "mid:old", []byte(postMsg), nil)
now = now.Add(time.Second)
s.EnqueuePost(dev.ID, "mid:kept", []byte(postMsg), nil)
kept := now
now = now.Add(400 * 24 * time.Hour) // the first is now older than 400 days, the second exactly 400 days old
hid, _ := s.HoldMessage(ops.ID, "a@x.com", []byte("x"), "r")
if err := s.EnqueueFromHeld(hid, "mid:new", []byte(postMsg), nil); err != nil {
t.Fatal(err)
}
posts, _ := s.ListPosts(dev.ID, 0)
if len(posts) != 1 || !posts[0].CreatedAt.Equal(kept) {
t.Fatalf("posts: %+v", posts)
}
now = now.Add(400 * 24 * time.Hour)
s.EnqueuePost(ops.ID, "mid:newer", []byte(postMsg), nil)
if posts, _ := s.ListPosts(dev.ID, 0); len(posts) != 0 {
t.Fatalf("EnqueuePost purges too: %+v", posts)
}
}
func TestDeletingAListDeletesItsPosts(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), nil)
if err := s.DeleteList(l.ID); err != nil {
t.Fatal(err)
}
var n int
if err := s.db.QueryRow(`SELECT COUNT(*) FROM posts`).Scan(&n); err != nil || n != 0 {
t.Fatalf("posts left: %d %v", n, err)
}
}
func TestMigrationAddsPostAndPriorityColumns(t *testing.T) {
path := t.TempDir() + "/old.db"
s, _ := Open(path)
s.Enqueue([]byte("x"), items(1))
for _, q := range []string{`DROP INDEX IF EXISTS outbox_due_prio`, `ALTER TABLE outbox DROP COLUMN post_id`,
`ALTER TABLE outbox DROP COLUMN priority`, `DROP TABLE posts`} {
if _, err := s.db.Exec(q); err != nil {
t.Fatalf("%s: %v", q, err)
}
}
s.Close()
s2, err := Open(path)
if err != nil {
t.Fatalf("migrate: %v", err)
}
defer s2.Close()
due, _ := s2.DueOutbox(10)
if len(due) != 1 {
t.Fatalf("an existing delivery stays waiting: %+v", due)
}
if ids := outboxPostIDs(t, s2); len(ids) != 1 || ids[0] != 0 {
t.Fatalf("post ids: %v", ids)
}
l, _ := s2.CreateList("dev", "example.org", "Dev")
if err := s2.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), items(1)); err != nil {
t.Fatal(err)
}
onlyPost(t, s2, l.ID)
}
func TestDueOutboxSendsLowerPriorityNumbersFirst(t *testing.T) {
s := newTestStore(t)
now := time.Unix(1_000_000, 0)
s.Now = func() time.Time { return now }
news := items(3)
for i := range news {
news[i].Priority = 1
}
s.Enqueue([]byte("newsletter"), news)
now = now.Add(time.Minute)
s.Enqueue([]byte("discussion"), items(2))
due, _ := s.DueOutbox(10)
if len(due) != 5 {
t.Fatalf("due: %d", len(due))
}
for i, d := range due {
if d.Priority != map[bool]int{true: 0, false: 1}[i < 2] {
t.Fatalf("priority 0 first: %+v", due)
}
}
for i := 1; i < len(due); i++ {
if due[i].Priority == due[i-1].Priority && due[i].ID < due[i-1].ID {
t.Fatalf("FIFO within a priority: %+v", due)
}
}
if got, _ := s.DueOutbox(2); got[0].Priority != 0 || got[1].Priority != 0 {
t.Fatalf("the limit takes the first in priority: %+v", got)
}
// A priority-0 delivery that is not due yet does not jump ahead.
s.RetryOutbox(due[0].ID, now.Add(time.Hour), "down")
s.RetryOutbox(due[1].ID, now.Add(time.Hour), "down")
if got, _ := s.DueOutbox(10); len(got) != 3 || got[0].Priority != 1 {
t.Fatalf("due: %+v", got)
}
}
func TestWithinAPriorityTheLongestWaitingGoesFirst(t *testing.T) {
s := newTestStore(t)
now := time.Unix(1_000_000, 0)
s.Now = func() time.Time { return now }
s.Enqueue([]byte("body"), items(3))
due, _ := s.DueOutbox(10)
s.RetryOutbox(due[2].ID, now.Add(-time.Hour), "e")
got, _ := s.DueOutbox(10)
if got[0].ID != due[2].ID || got[1].ID != due[0].ID || got[2].ID != due[1].ID {
t.Fatalf("order: %+v", got)
}
}
func TestFailingAFailedDeliveryAgainCountsItOnce(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), items(1))
due, _ := s.DueOutbox(10)
s.FailOutbox(due[0].ID, "refused")
if err := s.FailOutbox(due[0].ID, "refused again"); err != nil {
t.Fatal(err)
}
if p := onlyPost(t, s, l.ID); p.Failed != 1 || p.Pending != 0 {
t.Fatalf("post: %+v", p)
}
}