- posts table: one row per post queued to a list (message id, subject, time, recipients, sent, failed, dropped), deleted with the list and forgotten after 400 days when a new post is recorded - EnqueuePost and EnqueueFromHeld record the post in the same transaction and stamp every delivery with its id; a refused duplicate records nothing; other mail keeps post_id 0 - CompleteOutbox counts sent, the new DropOutbox counts dropped and FailOutbox counts failed, each in the transaction that changes the delivery; a delivery that already failed is never counted twice - outbox priority column; DueOutbox orders by priority, next_at, id - ListPosts newest first (default 50, at most 200) with Pending derived - additive migration of the outbox columns, tested on an old database
421 lines
13 KiB
Go
421 lines
13 KiB
Go
package store
|
|
|
|
import (
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
const postMsg = "Message-ID: <a@x>\r\nSubject: =?utf-8?q?Caff=C3=A8?= news\x07\r\n\r\nbody"
|
|
|
|
func onlyPost(t *testing.T, s *Store, listID int64) Post {
|
|
t.Helper()
|
|
posts, err := s.ListPosts(listID, 0)
|
|
if err != nil || len(posts) != 1 {
|
|
t.Fatalf("posts: %+v %v", posts, err)
|
|
}
|
|
return posts[0]
|
|
}
|
|
|
|
func outboxPostIDs(t *testing.T, s *Store) []int64 {
|
|
t.Helper()
|
|
rows, err := s.db.Query(`SELECT post_id FROM outbox ORDER BY id`)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer rows.Close()
|
|
var ids []int64
|
|
for rows.Next() {
|
|
var id int64
|
|
rows.Scan(&id)
|
|
ids = append(ids, id)
|
|
}
|
|
return ids
|
|
}
|
|
|
|
func TestEnqueuePostRecordsThePost(t *testing.T) {
|
|
s := newTestStore(t)
|
|
now := time.Unix(1_000_000, 0)
|
|
s.Now = func() time.Time { return now }
|
|
l, _ := s.CreateList("dev", "example.org", "Dev")
|
|
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), items(3)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
p := onlyPost(t, s, l.ID)
|
|
if p.ID == 0 || p.ListID != l.ID || p.MessageID != "<a@x>" || p.Subject != "Caffè news" || !p.CreatedAt.Equal(now) ||
|
|
p.Recipients != 3 || p.Sent != 0 || p.Failed != 0 || p.Dropped != 0 || p.Pending != 3 {
|
|
t.Fatalf("post: %+v", p)
|
|
}
|
|
for _, id := range outboxPostIDs(t, s) {
|
|
if id != p.ID {
|
|
t.Fatalf("every delivery carries the post id: %d != %d", id, p.ID)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestEnqueuePostRecordsAPostRowWithNoRecipients(t *testing.T) {
|
|
s := newTestStore(t)
|
|
l, _ := s.CreateList("dev", "example.org", "Dev")
|
|
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if p := onlyPost(t, s, l.ID); p.Recipients != 0 || p.Pending != 0 {
|
|
t.Fatalf("post: %+v", p)
|
|
}
|
|
}
|
|
|
|
func TestARefusedDuplicateRecordsNoPost(t *testing.T) {
|
|
s := newTestStore(t)
|
|
l, _ := s.CreateList("dev", "example.org", "Dev")
|
|
s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), items(1))
|
|
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), items(1)); !errors.Is(err, ErrDuplicate) {
|
|
t.Fatalf("want ErrDuplicate, got %v", err)
|
|
}
|
|
onlyPost(t, s, l.ID)
|
|
}
|
|
|
|
func TestAFailedQueueWriteRecordsNoPost(t *testing.T) {
|
|
s := newTestStore(t)
|
|
l, _ := s.CreateList("dev", "example.org", "Dev")
|
|
if _, err := s.db.Exec(`CREATE TRIGGER boom BEFORE INSERT ON outbox BEGIN SELECT RAISE(ABORT, 'boom'); END`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), items(2)); err == nil {
|
|
t.Fatal("want the queue error")
|
|
}
|
|
hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r")
|
|
if err := s.EnqueueFromHeld(hid, "mid:<h@x>", []byte(postMsg), items(2)); err == nil {
|
|
t.Fatal("want the queue error")
|
|
}
|
|
if posts, _ := s.ListPosts(l.ID, 0); len(posts) != 0 {
|
|
t.Fatalf("posts: %+v", posts)
|
|
}
|
|
}
|
|
|
|
func TestReleasingAHeldPostRecordsItOnce(t *testing.T) {
|
|
s := newTestStore(t)
|
|
l, _ := s.CreateList("dev", "example.org", "Dev")
|
|
hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r")
|
|
if err := s.EnqueueFromHeld(hid, "mid:<a@x>", []byte(postMsg), items(2)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s.EnqueueFromHeld(hid, "mid:<a@x>", []byte(postMsg), items(2)) // a second approval: ErrNotFound
|
|
p := onlyPost(t, s, l.ID)
|
|
if p.Recipients != 2 || p.Subject != "Caffè news" {
|
|
t.Fatalf("post: %+v", p)
|
|
}
|
|
for _, id := range outboxPostIDs(t, s) {
|
|
if id != p.ID {
|
|
t.Fatalf("delivery post id %d != %d", id, p.ID)
|
|
}
|
|
}
|
|
hid2, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r")
|
|
if err := s.EnqueueFromHeld(hid2, "mid:<b@x>", []byte(postMsg), nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if posts, _ := s.ListPosts(l.ID, 0); len(posts) != 2 || posts[0].Recipients != 0 {
|
|
t.Fatalf("a released post with no recipients has a row too: %+v", posts)
|
|
}
|
|
}
|
|
|
|
func TestOtherMailIsNotAPost(t *testing.T) {
|
|
s := newTestStore(t)
|
|
l, _ := s.CreateList("dev", "example.org", "Dev")
|
|
if err := s.Enqueue([]byte(postMsg), items(2)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if posts, _ := s.ListPosts(l.ID, 0); len(posts) != 0 {
|
|
t.Fatalf("posts: %+v", posts)
|
|
}
|
|
for _, id := range outboxPostIDs(t, s) {
|
|
if id != 0 {
|
|
t.Fatalf("post id %d", id)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPostSubjectIsCutAndClean(t *testing.T) {
|
|
s := newTestStore(t)
|
|
l, _ := s.CreateList("dev", "example.org", "Dev")
|
|
msg := "Message-ID: <a\x1b@x>\r\nSubject: " + strings.Repeat("é", 300) + "\r\n\r\nb"
|
|
s.EnqueuePost(l.ID, "mid:x", []byte(msg), items(1))
|
|
p := onlyPost(t, s, l.ID)
|
|
if p.Subject != strings.Repeat("é", 200) || p.MessageID != "<a@x>" {
|
|
t.Fatalf("post: %+v", p)
|
|
}
|
|
}
|
|
|
|
func TestCountersFollowTheDeliveries(t *testing.T) {
|
|
s := newTestStore(t)
|
|
now := time.Unix(1_000_000, 0)
|
|
s.Now = func() time.Time { return now }
|
|
l, _ := s.CreateList("dev", "example.org", "Dev")
|
|
s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), items(6))
|
|
due, _ := s.DueOutbox(10)
|
|
|
|
if err := s.RetryOutbox(due[0].ID, now.Add(time.Minute), "down"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if p := onlyPost(t, s, l.ID); p.Sent+p.Failed+p.Dropped != 0 || p.Pending != 6 {
|
|
t.Fatalf("a retry changes nothing: %+v", p)
|
|
}
|
|
if err := s.CompleteOutbox(due[1].ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.FailOutbox(due[2].ID, "refused"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.DropOutbox(due[3].ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.DropOutbox(due[4].ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
p := onlyPost(t, s, l.ID)
|
|
if p.Sent != 1 || p.Failed != 1 || p.Dropped != 2 || p.Pending != 2 {
|
|
t.Fatalf("post: %+v", p)
|
|
}
|
|
// Discarding a failed delivery: it is already counted as failed.
|
|
if err := s.DropOutbox(due[2].ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.CompleteOutbox(due[2].ID); !errors.Is(err, ErrNotFound) {
|
|
t.Fatalf("want ErrNotFound, got %v", err)
|
|
}
|
|
if err := s.DropOutbox(due[2].ID); !errors.Is(err, ErrNotFound) {
|
|
t.Fatalf("want ErrNotFound, got %v", err)
|
|
}
|
|
if p2 := onlyPost(t, s, l.ID); p2 != p {
|
|
t.Fatalf("discarding a failed delivery changes nothing: %+v", p2)
|
|
}
|
|
}
|
|
|
|
func TestCompletingAFailedDeliveryDoesNotCountItAsSent(t *testing.T) {
|
|
s := newTestStore(t)
|
|
l, _ := s.CreateList("dev", "example.org", "Dev")
|
|
s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), items(1))
|
|
due, _ := s.DueOutbox(10)
|
|
s.FailOutbox(due[0].ID, "refused")
|
|
if err := s.CompleteOutbox(due[0].ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if p := onlyPost(t, s, l.ID); p.Sent != 0 || p.Failed != 1 || p.Pending != 0 {
|
|
t.Fatalf("post: %+v", p)
|
|
}
|
|
}
|
|
|
|
func TestDropOutboxRemovesTheDeliveryAndTheBody(t *testing.T) {
|
|
s := newTestStore(t)
|
|
s.Enqueue([]byte("body"), items(1))
|
|
due, _ := s.DueOutbox(10)
|
|
if err := s.DropOutbox(due[0].ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if st, _ := s.OutboxStats(); st.Pending != 0 || st.Failed != 0 {
|
|
t.Fatalf("stats: %+v", st)
|
|
}
|
|
if _, err := s.OutboxData(due[0].MsgID); !errors.Is(err, ErrNotFound) {
|
|
t.Fatalf("body: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestPurgingOldFailuresKeepsTheCounters(t *testing.T) {
|
|
s := newTestStore(t)
|
|
now := time.Unix(1_000_000, 0)
|
|
s.Now = func() time.Time { return now }
|
|
l, _ := s.CreateList("dev", "example.org", "Dev")
|
|
s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), items(2))
|
|
due, _ := s.DueOutbox(10)
|
|
s.FailOutbox(due[0].ID, "refused")
|
|
now = now.Add(8 * 24 * time.Hour)
|
|
if n, err := s.PurgeFailedOutbox(); err != nil || n != 1 {
|
|
t.Fatalf("purged %d %v", n, err)
|
|
}
|
|
if p := onlyPost(t, s, l.ID); p.Failed != 1 || p.Pending != 1 {
|
|
t.Fatalf("post: %+v", p)
|
|
}
|
|
}
|
|
|
|
func TestListPostsNewestFirstWithALimit(t *testing.T) {
|
|
s := newTestStore(t)
|
|
now := time.Unix(1_000_000, 0)
|
|
s.Now = func() time.Time { return now }
|
|
dev, _ := s.CreateList("dev", "example.org", "Dev")
|
|
ops, _ := s.CreateList("ops", "example.org", "Ops")
|
|
for i := 0; i < 210; i++ {
|
|
now = now.Add(time.Second)
|
|
if err := s.EnqueuePost(dev.ID, "mid:"+string(rune(0x100+i)), []byte(postMsg), nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
s.EnqueuePost(ops.ID, "mid:ops", []byte(postMsg), items(1))
|
|
all, _ := s.ListPosts(dev.ID, 1000)
|
|
if len(all) != 200 {
|
|
t.Fatalf("capped at 200: %d", len(all))
|
|
}
|
|
for i := 1; i < len(all); i++ {
|
|
if !all[i-1].CreatedAt.After(all[i].CreatedAt) {
|
|
t.Fatalf("not newest first at %d", i)
|
|
}
|
|
}
|
|
if def, _ := s.ListPosts(dev.ID, 0); len(def) != 50 || def[0].ID != all[0].ID {
|
|
t.Fatalf("default 50: %d", len(def))
|
|
}
|
|
if neg, _ := s.ListPosts(dev.ID, -1); len(neg) != 50 {
|
|
t.Fatalf("default 50: %d", len(neg))
|
|
}
|
|
if some, _ := s.ListPosts(dev.ID, 7); len(some) != 7 {
|
|
t.Fatalf("limit 7: %d", len(some))
|
|
}
|
|
if exact, _ := s.ListPosts(dev.ID, 200); len(exact) != 200 {
|
|
t.Fatalf("limit 200: %d", len(exact))
|
|
}
|
|
if o, _ := s.ListPosts(ops.ID, 0); len(o) != 1 || o[0].Recipients != 1 {
|
|
t.Fatalf("other list: %+v", o)
|
|
}
|
|
}
|
|
|
|
func TestListPostsBreaksTiesByID(t *testing.T) {
|
|
s := newTestStore(t)
|
|
l, _ := s.CreateList("dev", "example.org", "Dev")
|
|
s.Now = func() time.Time { return time.Unix(1_000_000, 0) }
|
|
s.EnqueuePost(l.ID, "mid:1", []byte(postMsg), nil)
|
|
s.EnqueuePost(l.ID, "mid:2", []byte(postMsg), nil)
|
|
posts, _ := s.ListPosts(l.ID, 0)
|
|
if len(posts) != 2 || posts[0].ID < posts[1].ID {
|
|
t.Fatalf("the later post first: %+v", posts)
|
|
}
|
|
}
|
|
|
|
func TestOldPostsArePurgedWhenANewOneIsRecorded(t *testing.T) {
|
|
s := newTestStore(t)
|
|
now := time.Unix(100_000_000, 0)
|
|
s.Now = func() time.Time { return now }
|
|
dev, _ := s.CreateList("dev", "example.org", "Dev")
|
|
ops, _ := s.CreateList("ops", "example.org", "Ops")
|
|
s.EnqueuePost(dev.ID, "mid:old", []byte(postMsg), nil)
|
|
now = now.Add(time.Second)
|
|
s.EnqueuePost(dev.ID, "mid:kept", []byte(postMsg), nil)
|
|
kept := now
|
|
now = now.Add(400 * 24 * time.Hour) // the first is now older than 400 days, the second exactly 400 days old
|
|
hid, _ := s.HoldMessage(ops.ID, "a@x.com", []byte("x"), "r")
|
|
if err := s.EnqueueFromHeld(hid, "mid:new", []byte(postMsg), nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
posts, _ := s.ListPosts(dev.ID, 0)
|
|
if len(posts) != 1 || !posts[0].CreatedAt.Equal(kept) {
|
|
t.Fatalf("posts: %+v", posts)
|
|
}
|
|
now = now.Add(400 * 24 * time.Hour)
|
|
s.EnqueuePost(ops.ID, "mid:newer", []byte(postMsg), nil)
|
|
if posts, _ := s.ListPosts(dev.ID, 0); len(posts) != 0 {
|
|
t.Fatalf("EnqueuePost purges too: %+v", posts)
|
|
}
|
|
}
|
|
|
|
func TestDeletingAListDeletesItsPosts(t *testing.T) {
|
|
s := newTestStore(t)
|
|
l, _ := s.CreateList("dev", "example.org", "Dev")
|
|
s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), nil)
|
|
if err := s.DeleteList(l.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var n int
|
|
if err := s.db.QueryRow(`SELECT COUNT(*) FROM posts`).Scan(&n); err != nil || n != 0 {
|
|
t.Fatalf("posts left: %d %v", n, err)
|
|
}
|
|
}
|
|
|
|
func TestMigrationAddsPostAndPriorityColumns(t *testing.T) {
|
|
path := t.TempDir() + "/old.db"
|
|
s, _ := Open(path)
|
|
s.Enqueue([]byte("x"), items(1))
|
|
for _, q := range []string{`DROP INDEX IF EXISTS outbox_due_prio`, `ALTER TABLE outbox DROP COLUMN post_id`,
|
|
`ALTER TABLE outbox DROP COLUMN priority`, `DROP TABLE posts`} {
|
|
if _, err := s.db.Exec(q); err != nil {
|
|
t.Fatalf("%s: %v", q, err)
|
|
}
|
|
}
|
|
s.Close()
|
|
s2, err := Open(path)
|
|
if err != nil {
|
|
t.Fatalf("migrate: %v", err)
|
|
}
|
|
defer s2.Close()
|
|
due, _ := s2.DueOutbox(10)
|
|
if len(due) != 1 {
|
|
t.Fatalf("an existing delivery stays waiting: %+v", due)
|
|
}
|
|
if ids := outboxPostIDs(t, s2); len(ids) != 1 || ids[0] != 0 {
|
|
t.Fatalf("post ids: %v", ids)
|
|
}
|
|
l, _ := s2.CreateList("dev", "example.org", "Dev")
|
|
if err := s2.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), items(1)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
onlyPost(t, s2, l.ID)
|
|
}
|
|
|
|
func TestDueOutboxSendsLowerPriorityNumbersFirst(t *testing.T) {
|
|
s := newTestStore(t)
|
|
now := time.Unix(1_000_000, 0)
|
|
s.Now = func() time.Time { return now }
|
|
news := items(3)
|
|
for i := range news {
|
|
news[i].Priority = 1
|
|
}
|
|
s.Enqueue([]byte("newsletter"), news)
|
|
now = now.Add(time.Minute)
|
|
s.Enqueue([]byte("discussion"), items(2))
|
|
due, _ := s.DueOutbox(10)
|
|
if len(due) != 5 {
|
|
t.Fatalf("due: %d", len(due))
|
|
}
|
|
for i, d := range due {
|
|
if d.Priority != map[bool]int{true: 0, false: 1}[i < 2] {
|
|
t.Fatalf("priority 0 first: %+v", due)
|
|
}
|
|
}
|
|
for i := 1; i < len(due); i++ {
|
|
if due[i].Priority == due[i-1].Priority && due[i].ID < due[i-1].ID {
|
|
t.Fatalf("FIFO within a priority: %+v", due)
|
|
}
|
|
}
|
|
if got, _ := s.DueOutbox(2); got[0].Priority != 0 || got[1].Priority != 0 {
|
|
t.Fatalf("the limit takes the first in priority: %+v", got)
|
|
}
|
|
// A priority-0 delivery that is not due yet does not jump ahead.
|
|
s.RetryOutbox(due[0].ID, now.Add(time.Hour), "down")
|
|
s.RetryOutbox(due[1].ID, now.Add(time.Hour), "down")
|
|
if got, _ := s.DueOutbox(10); len(got) != 3 || got[0].Priority != 1 {
|
|
t.Fatalf("due: %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestWithinAPriorityTheLongestWaitingGoesFirst(t *testing.T) {
|
|
s := newTestStore(t)
|
|
now := time.Unix(1_000_000, 0)
|
|
s.Now = func() time.Time { return now }
|
|
s.Enqueue([]byte("body"), items(3))
|
|
due, _ := s.DueOutbox(10)
|
|
s.RetryOutbox(due[2].ID, now.Add(-time.Hour), "e")
|
|
got, _ := s.DueOutbox(10)
|
|
if got[0].ID != due[2].ID || got[1].ID != due[0].ID || got[2].ID != due[1].ID {
|
|
t.Fatalf("order: %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestFailingAFailedDeliveryAgainCountsItOnce(t *testing.T) {
|
|
s := newTestStore(t)
|
|
l, _ := s.CreateList("dev", "example.org", "Dev")
|
|
s.EnqueuePost(l.ID, "mid:<a@x>", []byte(postMsg), items(1))
|
|
due, _ := s.DueOutbox(10)
|
|
s.FailOutbox(due[0].ID, "refused")
|
|
if err := s.FailOutbox(due[0].ID, "refused again"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if p := onlyPost(t, s, l.ID); p.Failed != 1 || p.Pending != 0 {
|
|
t.Fatalf("post: %+v", p)
|
|
}
|
|
}
|