diff --git a/internal/store/outbox.go b/internal/store/outbox.go index c72abea..a0aea3b 100644 --- a/internal/store/outbox.go +++ b/internal/store/outbox.go @@ -12,6 +12,7 @@ type NewOutbound struct { Rcpt string MemberID int64 // 0 for system mail Prefix []byte // per-recipient headers, sent before the shared message + Priority int // lower goes first among due deliveries: 0 for discussion and system mail, 1 for a newsletter } // OutboxItem is a queued delivery to one recipient. @@ -28,7 +29,8 @@ type OutboxItem struct { CreatedAt time.Time // Failed deliveries are never retried (the relay refused them for good, or they are too old). // They stay listed for a week so the admin can see what was lost and why. - Failed bool + Failed bool + Priority int } // OutboxStats summarises the queue. @@ -52,7 +54,7 @@ func (s *Store) Enqueue(msg []byte, items []NewOutbound) error { return err } defer tx.Rollback() - if err := enqueueTx(tx, s.Now().Unix(), msg, items); err != nil { + if err := enqueueTx(tx, s.Now().Unix(), 0, msg, items); err != nil { return err } return tx.Commit() @@ -65,6 +67,8 @@ const postMemory = time.Hour // in the same transaction, so a crash never leaves one without the other. If the list already // queued the same postID within postMemory it queues nothing and returns ErrDuplicate: the MTA // delivers the post again when our reply to the first delivery was lost. +// It also records the post (a Post, with its counters) and stamps every delivery with its id; +// a refused duplicate records nothing. func (s *Store) EnqueuePost(listID int64, postID string, msg []byte, items []NewOutbound) error { now := s.Now() tx, err := s.db.Begin() @@ -81,8 +85,12 @@ func (s *Store) EnqueuePost(listID int64, postID string, msg []byte, items []New } return err } + post, err := recordPostTx(tx, now, listID, msg, len(items)) + if err != nil { + return err + } if len(items) > 0 { - if err := enqueueTx(tx, now.Unix(), msg, items); err != nil { + if err := enqueueTx(tx, now.Unix(), post, msg, items); err != nil { return err } } @@ -93,7 +101,7 @@ func (s *Store) EnqueuePost(listID int64, postID string, msg []byte, items []New // its deliveries in one transaction. Of several simultaneous approvals only one succeeds, the // others get ErrNotFound, so a message is never delivered twice. With no recipients the message // is still released. An approval always delivers, even a post already seen, and records postID -// as EnqueuePost does, so a later copy of the same post is refused. +// as EnqueuePost does, so a later copy of the same post is refused. It records the post as EnqueuePost does. func (s *Store) EnqueueFromHeld(heldID int64, postID string, msg []byte, items []NewOutbound) error { tx, err := s.db.Begin() if err != nil { @@ -113,15 +121,20 @@ func (s *Store) EnqueueFromHeld(heldID int64, postID string, msg []byte, items [ ON CONFLICT(list_id, post_id) DO UPDATE SET created_at=excluded.created_at`, listID, postID, s.Now().Unix()); err != nil { return err } + post, err := recordPostTx(tx, s.Now(), listID, msg, len(items)) + if err != nil { + return err + } if len(items) > 0 { - if err := enqueueTx(tx, s.Now().Unix(), msg, items); err != nil { + if err := enqueueTx(tx, s.Now().Unix(), post, msg, items); err != nil { return err } } return tx.Commit() } -func enqueueTx(tx *sql.Tx, now int64, msg []byte, items []NewOutbound) error { +// enqueueTx stores msg and its deliveries; postID is the posts row they count for, 0 for other mail. +func enqueueTx(tx *sql.Tx, now, postID int64, msg []byte, items []NewOutbound) error { res, err := tx.Exec(`INSERT INTO outbox_msgs(data) VALUES(?)`, msg) if err != nil { return err @@ -132,20 +145,20 @@ func enqueueTx(tx *sql.Tx, now int64, msg []byte, items []NewOutbound) error { if prefix == nil { prefix = []byte{} } - if _, err := tx.Exec(`INSERT INTO outbox(msg_id, envelope_from, rcpt, member_id, prefix, next_at, created_at) - VALUES(?,?,?,?,?,?,?)`, msgID, it.From, it.Rcpt, it.MemberID, prefix, now, now); err != nil { + if _, err := tx.Exec(`INSERT INTO outbox(msg_id, envelope_from, rcpt, member_id, prefix, next_at, created_at, post_id, priority) + VALUES(?,?,?,?,?,?,?,?,?)`, msgID, it.From, it.Rcpt, it.MemberID, prefix, now, now, postID, it.Priority); err != nil { return err } } return nil } -const outboxCols = `id, msg_id, envelope_from, rcpt, member_id, prefix, attempts, next_at, last_error, created_at, failed` +const outboxCols = `id, msg_id, envelope_from, rcpt, member_id, prefix, attempts, next_at, last_error, created_at, failed, priority` func scanOutbox(row interface{ Scan(...any) error }) (OutboxItem, error) { var it OutboxItem var next, created, failed int64 - err := row.Scan(&it.ID, &it.MsgID, &it.From, &it.Rcpt, &it.MemberID, &it.Prefix, &it.Attempts, &next, &it.LastError, &created, &failed) + err := row.Scan(&it.ID, &it.MsgID, &it.From, &it.Rcpt, &it.MemberID, &it.Prefix, &it.Attempts, &next, &it.LastError, &created, &failed, &it.Priority) it.NextAt, it.CreatedAt, it.Failed = time.Unix(next, 0), time.Unix(created, 0), failed != 0 return it, err } @@ -167,9 +180,10 @@ func (s *Store) queryOutbox(q string, args ...any) ([]OutboxItem, error) { return out, rows.Err() } -// DueOutbox returns up to limit deliveries whose time has come, longest waiting first. +// DueOutbox returns up to limit deliveries whose time has come: lowest priority number first (so a +// big newsletter never delays discussion mail), then longest waiting first. func (s *Store) DueOutbox(limit int) ([]OutboxItem, error) { - return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox WHERE failed=0 AND next_at<=? ORDER BY next_at, id LIMIT ?`, + return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox WHERE failed=0 AND next_at<=? ORDER BY priority, next_at, id LIMIT ?`, s.Now().Unix(), limit) } @@ -188,16 +202,26 @@ func (s *Store) OutboxData(msgID int64) ([]byte, error) { return b, err } -// CompleteOutbox removes a delivery (sent, dropped or discarded) and the shared -// message once no recipient needs it. -func (s *Store) CompleteOutbox(id int64) error { +// CompleteOutbox removes a delivery the relay accepted, and the shared message once no recipient +// needs it. A delivery of a post counts as sent, in the same transaction. +func (s *Store) CompleteOutbox(id int64) error { return s.removeOutbox(id, "sent") } + +// DropOutbox removes a delivery that will never be made (the mailbox does not exist, or the admin +// discarded it), like CompleteOutbox. A delivery of a post counts as dropped, unless it had +// already failed: it is counted as failed already. +func (s *Store) DropOutbox(id int64) error { return s.removeOutbox(id, "dropped") } + +// removeOutbox deletes a delivery and adds one to counter (a column of posts, never user input) +// of its post, when it has one and it had not failed. +func (s *Store) removeOutbox(id int64, counter string) error { tx, err := s.db.Begin() if err != nil { return err } defer tx.Rollback() - var msgID int64 - err = tx.QueryRow(`SELECT msg_id FROM outbox WHERE id=?`, id).Scan(&msgID) + var msgID, postID int64 + var failed bool + err = tx.QueryRow(`SELECT msg_id, post_id, failed FROM outbox WHERE id=?`, id).Scan(&msgID, &postID, &failed) if errors.Is(err, sql.ErrNoRows) { return ErrNotFound } @@ -210,6 +234,11 @@ func (s *Store) CompleteOutbox(id int64) error { if _, err := tx.Exec(`DELETE FROM outbox_msgs WHERE id=? AND NOT EXISTS (SELECT 1 FROM outbox WHERE msg_id=?)`, msgID, msgID); err != nil { return err } + if postID != 0 && !failed { + if _, err := tx.Exec(`UPDATE posts SET `+counter+`=`+counter+`+1 WHERE id=?`, postID); err != nil { + return err + } + } return tx.Commit() } @@ -232,10 +261,33 @@ func (s *Store) OutboxStats() (OutboxStats, error) { } // FailOutbox gives up on a delivery for good, keeping it listed with the reason. Its time is the -// moment of the failure, from which PurgeFailedOutbox counts. +// moment of the failure, from which PurgeFailedOutbox counts. A delivery of a post counts as failed, +// in the same transaction. func (s *Store) FailOutbox(id int64, reason string) error { - return affected(s.db.Exec(`UPDATE outbox SET failed=1, attempts=attempts+1, last_error=?, next_at=? WHERE id=?`, - reason, s.Now().Unix(), id)) + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + var postID int64 + var failed bool + err = tx.QueryRow(`SELECT post_id, failed FROM outbox WHERE id=?`, id).Scan(&postID, &failed) + if errors.Is(err, sql.ErrNoRows) { + return ErrNotFound + } + if err != nil { + return err + } + if _, err := tx.Exec(`UPDATE outbox SET failed=1, attempts=attempts+1, last_error=?, next_at=? WHERE id=?`, + reason, s.Now().Unix(), id); err != nil { + return err + } + if postID != 0 && !failed { + if _, err := tx.Exec(`UPDATE posts SET failed=failed+1 WHERE id=?`, postID); err != nil { + return err + } + } + return tx.Commit() } // PurgeFailedOutbox forgets failed deliveries older than a week, and the stored messages nobody diff --git a/internal/store/posts.go b/internal/store/posts.go new file mode 100644 index 0000000..e9f3323 --- /dev/null +++ b/internal/store/posts.go @@ -0,0 +1,78 @@ +package store + +import ( + "bytes" + "database/sql" + "net/mail" + "time" +) + +// Post is a post queued to a list, with the outcome of its deliveries so far. +type Post struct { + ID int64 + ListID int64 + MessageID string // the Message-ID header, at most 200 characters + Subject string // decoded, at most 200 characters + CreatedAt time.Time + Recipients int // deliveries queued + Sent int // accepted by the relay + Failed int // given up for good + Dropped int // the mailbox does not exist, or discarded by the admin + Pending int // still in the queue: Recipients - Sent - Failed - Dropped +} + +// postTTL is how long a post and its counters are kept. +const postTTL = 400 * 24 * time.Hour + +// Limits of ListPosts. +const ( + defaultPostsLimit = 50 + maxPostsLimit = 200 +) + +// recordPostTx stores the posts row of a post queued to a list and returns its id. It forgets +// posts older than postTTL on the way, so the table never grows without bound. +func recordPostTx(tx *sql.Tx, now time.Time, listID int64, msg []byte, recipients int) (int64, error) { + if _, err := tx.Exec(`DELETE FROM posts WHERE created_at maxPostsLimit { + limit = maxPostsLimit + } + rows, err := s.db.Query(`SELECT id, list_id, message_id, subject, created_at, recipients, sent, failed, dropped + FROM posts WHERE list_id=? ORDER BY created_at DESC, id DESC LIMIT ?`, listID, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Post + for rows.Next() { + var p Post + var created int64 + if err := rows.Scan(&p.ID, &p.ListID, &p.MessageID, &p.Subject, &created, &p.Recipients, &p.Sent, &p.Failed, &p.Dropped); err != nil { + return nil, err + } + p.CreatedAt = time.Unix(created, 0) + p.Pending = p.Recipients - p.Sent - p.Failed - p.Dropped + out = append(out, p) + } + return out, rows.Err() +} diff --git a/internal/store/posts_test.go b/internal/store/posts_test.go new file mode 100644 index 0000000..1c9bce1 --- /dev/null +++ b/internal/store/posts_test.go @@ -0,0 +1,421 @@ +package store + +import ( + "errors" + "strings" + "testing" + "time" +) + +const postMsg = "Message-ID: \r\nSubject: =?utf-8?q?Caff=C3=A8?= news\x07\r\n\r\nbody" + +func onlyPost(t *testing.T, s *Store, listID int64) Post { + t.Helper() + posts, err := s.ListPosts(listID, 0) + if err != nil || len(posts) != 1 { + t.Fatalf("posts: %+v %v", posts, err) + } + return posts[0] +} + +func outboxPostIDs(t *testing.T, s *Store) []int64 { + t.Helper() + rows, err := s.db.Query(`SELECT post_id FROM outbox ORDER BY id`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var ids []int64 + for rows.Next() { + var id int64 + rows.Scan(&id) + ids = append(ids, id) + } + return ids +} + +func TestEnqueuePostRecordsThePost(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + l, _ := s.CreateList("dev", "example.org", "Dev") + if err := s.EnqueuePost(l.ID, "mid:", []byte(postMsg), items(3)); err != nil { + t.Fatal(err) + } + p := onlyPost(t, s, l.ID) + if p.ID == 0 || p.ListID != l.ID || p.MessageID != "" || p.Subject != "Caffè news" || !p.CreatedAt.Equal(now) || + p.Recipients != 3 || p.Sent != 0 || p.Failed != 0 || p.Dropped != 0 || p.Pending != 3 { + t.Fatalf("post: %+v", p) + } + for _, id := range outboxPostIDs(t, s) { + if id != p.ID { + t.Fatalf("every delivery carries the post id: %d != %d", id, p.ID) + } + } +} + +func TestEnqueuePostRecordsAPostRowWithNoRecipients(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + if err := s.EnqueuePost(l.ID, "mid:", []byte(postMsg), nil); err != nil { + t.Fatal(err) + } + if p := onlyPost(t, s, l.ID); p.Recipients != 0 || p.Pending != 0 { + t.Fatalf("post: %+v", p) + } +} + +func TestARefusedDuplicateRecordsNoPost(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.EnqueuePost(l.ID, "mid:", []byte(postMsg), items(1)) + if err := s.EnqueuePost(l.ID, "mid:", []byte(postMsg), items(1)); !errors.Is(err, ErrDuplicate) { + t.Fatalf("want ErrDuplicate, got %v", err) + } + onlyPost(t, s, l.ID) +} + +func TestAFailedQueueWriteRecordsNoPost(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + if _, err := s.db.Exec(`CREATE TRIGGER boom BEFORE INSERT ON outbox BEGIN SELECT RAISE(ABORT, 'boom'); END`); err != nil { + t.Fatal(err) + } + if err := s.EnqueuePost(l.ID, "mid:", []byte(postMsg), items(2)); err == nil { + t.Fatal("want the queue error") + } + hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r") + if err := s.EnqueueFromHeld(hid, "mid:", []byte(postMsg), items(2)); err == nil { + t.Fatal("want the queue error") + } + if posts, _ := s.ListPosts(l.ID, 0); len(posts) != 0 { + t.Fatalf("posts: %+v", posts) + } +} + +func TestReleasingAHeldPostRecordsItOnce(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r") + if err := s.EnqueueFromHeld(hid, "mid:", []byte(postMsg), items(2)); err != nil { + t.Fatal(err) + } + s.EnqueueFromHeld(hid, "mid:", []byte(postMsg), items(2)) // a second approval: ErrNotFound + p := onlyPost(t, s, l.ID) + if p.Recipients != 2 || p.Subject != "Caffè news" { + t.Fatalf("post: %+v", p) + } + for _, id := range outboxPostIDs(t, s) { + if id != p.ID { + t.Fatalf("delivery post id %d != %d", id, p.ID) + } + } + hid2, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r") + if err := s.EnqueueFromHeld(hid2, "mid:", []byte(postMsg), nil); err != nil { + t.Fatal(err) + } + if posts, _ := s.ListPosts(l.ID, 0); len(posts) != 2 || posts[0].Recipients != 0 { + t.Fatalf("a released post with no recipients has a row too: %+v", posts) + } +} + +func TestOtherMailIsNotAPost(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + if err := s.Enqueue([]byte(postMsg), items(2)); err != nil { + t.Fatal(err) + } + if posts, _ := s.ListPosts(l.ID, 0); len(posts) != 0 { + t.Fatalf("posts: %+v", posts) + } + for _, id := range outboxPostIDs(t, s) { + if id != 0 { + t.Fatalf("post id %d", id) + } + } +} + +func TestPostSubjectIsCutAndClean(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + msg := "Message-ID: \r\nSubject: " + strings.Repeat("é", 300) + "\r\n\r\nb" + s.EnqueuePost(l.ID, "mid:x", []byte(msg), items(1)) + p := onlyPost(t, s, l.ID) + if p.Subject != strings.Repeat("é", 200) || p.MessageID != "" { + t.Fatalf("post: %+v", p) + } +} + +func TestCountersFollowTheDeliveries(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + l, _ := s.CreateList("dev", "example.org", "Dev") + s.EnqueuePost(l.ID, "mid:", []byte(postMsg), items(6)) + due, _ := s.DueOutbox(10) + + if err := s.RetryOutbox(due[0].ID, now.Add(time.Minute), "down"); err != nil { + t.Fatal(err) + } + if p := onlyPost(t, s, l.ID); p.Sent+p.Failed+p.Dropped != 0 || p.Pending != 6 { + t.Fatalf("a retry changes nothing: %+v", p) + } + if err := s.CompleteOutbox(due[1].ID); err != nil { + t.Fatal(err) + } + if err := s.FailOutbox(due[2].ID, "refused"); err != nil { + t.Fatal(err) + } + if err := s.DropOutbox(due[3].ID); err != nil { + t.Fatal(err) + } + if err := s.DropOutbox(due[4].ID); err != nil { + t.Fatal(err) + } + p := onlyPost(t, s, l.ID) + if p.Sent != 1 || p.Failed != 1 || p.Dropped != 2 || p.Pending != 2 { + t.Fatalf("post: %+v", p) + } + // Discarding a failed delivery: it is already counted as failed. + if err := s.DropOutbox(due[2].ID); err != nil { + t.Fatal(err) + } + if err := s.CompleteOutbox(due[2].ID); !errors.Is(err, ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } + if err := s.DropOutbox(due[2].ID); !errors.Is(err, ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } + if p2 := onlyPost(t, s, l.ID); p2 != p { + t.Fatalf("discarding a failed delivery changes nothing: %+v", p2) + } +} + +func TestCompletingAFailedDeliveryDoesNotCountItAsSent(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.EnqueuePost(l.ID, "mid:", []byte(postMsg), items(1)) + due, _ := s.DueOutbox(10) + s.FailOutbox(due[0].ID, "refused") + if err := s.CompleteOutbox(due[0].ID); err != nil { + t.Fatal(err) + } + if p := onlyPost(t, s, l.ID); p.Sent != 0 || p.Failed != 1 || p.Pending != 0 { + t.Fatalf("post: %+v", p) + } +} + +func TestDropOutboxRemovesTheDeliveryAndTheBody(t *testing.T) { + s := newTestStore(t) + s.Enqueue([]byte("body"), items(1)) + due, _ := s.DueOutbox(10) + if err := s.DropOutbox(due[0].ID); err != nil { + t.Fatal(err) + } + if st, _ := s.OutboxStats(); st.Pending != 0 || st.Failed != 0 { + t.Fatalf("stats: %+v", st) + } + if _, err := s.OutboxData(due[0].MsgID); !errors.Is(err, ErrNotFound) { + t.Fatalf("body: %v", err) + } +} + +func TestPurgingOldFailuresKeepsTheCounters(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + l, _ := s.CreateList("dev", "example.org", "Dev") + s.EnqueuePost(l.ID, "mid:", []byte(postMsg), items(2)) + due, _ := s.DueOutbox(10) + s.FailOutbox(due[0].ID, "refused") + now = now.Add(8 * 24 * time.Hour) + if n, err := s.PurgeFailedOutbox(); err != nil || n != 1 { + t.Fatalf("purged %d %v", n, err) + } + if p := onlyPost(t, s, l.ID); p.Failed != 1 || p.Pending != 1 { + t.Fatalf("post: %+v", p) + } +} + +func TestListPostsNewestFirstWithALimit(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + dev, _ := s.CreateList("dev", "example.org", "Dev") + ops, _ := s.CreateList("ops", "example.org", "Ops") + for i := 0; i < 210; i++ { + now = now.Add(time.Second) + if err := s.EnqueuePost(dev.ID, "mid:"+string(rune(0x100+i)), []byte(postMsg), nil); err != nil { + t.Fatal(err) + } + } + s.EnqueuePost(ops.ID, "mid:ops", []byte(postMsg), items(1)) + all, _ := s.ListPosts(dev.ID, 1000) + if len(all) != 200 { + t.Fatalf("capped at 200: %d", len(all)) + } + for i := 1; i < len(all); i++ { + if !all[i-1].CreatedAt.After(all[i].CreatedAt) { + t.Fatalf("not newest first at %d", i) + } + } + if def, _ := s.ListPosts(dev.ID, 0); len(def) != 50 || def[0].ID != all[0].ID { + t.Fatalf("default 50: %d", len(def)) + } + if neg, _ := s.ListPosts(dev.ID, -1); len(neg) != 50 { + t.Fatalf("default 50: %d", len(neg)) + } + if some, _ := s.ListPosts(dev.ID, 7); len(some) != 7 { + t.Fatalf("limit 7: %d", len(some)) + } + if exact, _ := s.ListPosts(dev.ID, 200); len(exact) != 200 { + t.Fatalf("limit 200: %d", len(exact)) + } + if o, _ := s.ListPosts(ops.ID, 0); len(o) != 1 || o[0].Recipients != 1 { + t.Fatalf("other list: %+v", o) + } +} + +func TestListPostsBreaksTiesByID(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.Now = func() time.Time { return time.Unix(1_000_000, 0) } + s.EnqueuePost(l.ID, "mid:1", []byte(postMsg), nil) + s.EnqueuePost(l.ID, "mid:2", []byte(postMsg), nil) + posts, _ := s.ListPosts(l.ID, 0) + if len(posts) != 2 || posts[0].ID < posts[1].ID { + t.Fatalf("the later post first: %+v", posts) + } +} + +func TestOldPostsArePurgedWhenANewOneIsRecorded(t *testing.T) { + s := newTestStore(t) + now := time.Unix(100_000_000, 0) + s.Now = func() time.Time { return now } + dev, _ := s.CreateList("dev", "example.org", "Dev") + ops, _ := s.CreateList("ops", "example.org", "Ops") + s.EnqueuePost(dev.ID, "mid:old", []byte(postMsg), nil) + now = now.Add(time.Second) + s.EnqueuePost(dev.ID, "mid:kept", []byte(postMsg), nil) + kept := now + now = now.Add(400 * 24 * time.Hour) // the first is now older than 400 days, the second exactly 400 days old + hid, _ := s.HoldMessage(ops.ID, "a@x.com", []byte("x"), "r") + if err := s.EnqueueFromHeld(hid, "mid:new", []byte(postMsg), nil); err != nil { + t.Fatal(err) + } + posts, _ := s.ListPosts(dev.ID, 0) + if len(posts) != 1 || !posts[0].CreatedAt.Equal(kept) { + t.Fatalf("posts: %+v", posts) + } + now = now.Add(400 * 24 * time.Hour) + s.EnqueuePost(ops.ID, "mid:newer", []byte(postMsg), nil) + if posts, _ := s.ListPosts(dev.ID, 0); len(posts) != 0 { + t.Fatalf("EnqueuePost purges too: %+v", posts) + } +} + +func TestDeletingAListDeletesItsPosts(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.EnqueuePost(l.ID, "mid:", []byte(postMsg), nil) + if err := s.DeleteList(l.ID); err != nil { + t.Fatal(err) + } + var n int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM posts`).Scan(&n); err != nil || n != 0 { + t.Fatalf("posts left: %d %v", n, err) + } +} + +func TestMigrationAddsPostAndPriorityColumns(t *testing.T) { + path := t.TempDir() + "/old.db" + s, _ := Open(path) + s.Enqueue([]byte("x"), items(1)) + for _, q := range []string{`DROP INDEX IF EXISTS outbox_due_prio`, `ALTER TABLE outbox DROP COLUMN post_id`, + `ALTER TABLE outbox DROP COLUMN priority`, `DROP TABLE posts`} { + if _, err := s.db.Exec(q); err != nil { + t.Fatalf("%s: %v", q, err) + } + } + s.Close() + s2, err := Open(path) + if err != nil { + t.Fatalf("migrate: %v", err) + } + defer s2.Close() + due, _ := s2.DueOutbox(10) + if len(due) != 1 { + t.Fatalf("an existing delivery stays waiting: %+v", due) + } + if ids := outboxPostIDs(t, s2); len(ids) != 1 || ids[0] != 0 { + t.Fatalf("post ids: %v", ids) + } + l, _ := s2.CreateList("dev", "example.org", "Dev") + if err := s2.EnqueuePost(l.ID, "mid:", []byte(postMsg), items(1)); err != nil { + t.Fatal(err) + } + onlyPost(t, s2, l.ID) +} + +func TestDueOutboxSendsLowerPriorityNumbersFirst(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + news := items(3) + for i := range news { + news[i].Priority = 1 + } + s.Enqueue([]byte("newsletter"), news) + now = now.Add(time.Minute) + s.Enqueue([]byte("discussion"), items(2)) + due, _ := s.DueOutbox(10) + if len(due) != 5 { + t.Fatalf("due: %d", len(due)) + } + for i, d := range due { + if d.Priority != map[bool]int{true: 0, false: 1}[i < 2] { + t.Fatalf("priority 0 first: %+v", due) + } + } + for i := 1; i < len(due); i++ { + if due[i].Priority == due[i-1].Priority && due[i].ID < due[i-1].ID { + t.Fatalf("FIFO within a priority: %+v", due) + } + } + if got, _ := s.DueOutbox(2); got[0].Priority != 0 || got[1].Priority != 0 { + t.Fatalf("the limit takes the first in priority: %+v", got) + } + // A priority-0 delivery that is not due yet does not jump ahead. + s.RetryOutbox(due[0].ID, now.Add(time.Hour), "down") + s.RetryOutbox(due[1].ID, now.Add(time.Hour), "down") + if got, _ := s.DueOutbox(10); len(got) != 3 || got[0].Priority != 1 { + t.Fatalf("due: %+v", got) + } +} + +func TestWithinAPriorityTheLongestWaitingGoesFirst(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + s.Enqueue([]byte("body"), items(3)) + due, _ := s.DueOutbox(10) + s.RetryOutbox(due[2].ID, now.Add(-time.Hour), "e") + got, _ := s.DueOutbox(10) + if got[0].ID != due[2].ID || got[1].ID != due[0].ID || got[2].ID != due[1].ID { + t.Fatalf("order: %+v", got) + } +} + +func TestFailingAFailedDeliveryAgainCountsItOnce(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.EnqueuePost(l.ID, "mid:", []byte(postMsg), items(1)) + due, _ := s.DueOutbox(10) + s.FailOutbox(due[0].ID, "refused") + if err := s.FailOutbox(due[0].ID, "refused again"); err != nil { + t.Fatal(err) + } + if p := onlyPost(t, s, l.ID); p.Failed != 1 || p.Pending != 0 { + t.Fatalf("post: %+v", p) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 705bb1a..a046d4a 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -212,9 +212,24 @@ CREATE TABLE IF NOT EXISTS outbox ( next_at INTEGER NOT NULL, last_error TEXT NOT NULL DEFAULT '', created_at INTEGER NOT NULL, - failed INTEGER NOT NULL DEFAULT 0 + failed INTEGER NOT NULL DEFAULT 0, + post_id INTEGER NOT NULL DEFAULT 0, + priority INTEGER NOT NULL DEFAULT 0 ); CREATE INDEX IF NOT EXISTS outbox_due ON outbox(next_at, id); +CREATE INDEX IF NOT EXISTS outbox_due_prio ON outbox(priority, next_at, id); +CREATE TABLE IF NOT EXISTS posts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE, + message_id TEXT NOT NULL DEFAULT '', + subject TEXT NOT NULL DEFAULT '', + created_at INTEGER NOT NULL, + recipients INTEGER NOT NULL DEFAULT 0, + sent INTEGER NOT NULL DEFAULT 0, + failed INTEGER NOT NULL DEFAULT 0, + dropped INTEGER NOT NULL DEFAULT 0 +); +CREATE INDEX IF NOT EXISTS posts_list ON posts(list_id, created_at, id); CREATE TABLE IF NOT EXISTS delivered ( list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE, post_id TEXT NOT NULL, @@ -302,6 +317,12 @@ func migrate(db *sql.DB) error { return err } } + if has, _ := hasColumn(db, "outbox", "post_id"); !has { + if _, err := db.Exec(`ALTER TABLE outbox ADD COLUMN post_id INTEGER NOT NULL DEFAULT 0; + ALTER TABLE outbox ADD COLUMN priority INTEGER NOT NULL DEFAULT 0;`); err != nil { + return err + } + } } if ok, err := hasColumn(db, "held", "list_id"); err != nil { return err @@ -390,23 +411,25 @@ func summarize(data []byte) (subject, from string) { if err != nil { return "", "" } - clean := func(s string) string { - s = strings.Map(func(r rune) rune { - if unicode.IsControl(r) { - return -1 - } - return r - }, strings.Join(strings.Fields(s), " ")) - if r := []rune(s); len(r) > 200 { - s = string(r[:200]) - } - return s - } raw := m.Header.Get("Subject") if dec, err := new(mime.WordDecoder).DecodeHeader(raw); err == nil { raw = dec } - return clean(raw), clean(m.Header.Get("From")) + return cleanSummary(raw), cleanSummary(m.Header.Get("From")) +} + +// cleanSummary makes a header value fit for a listing: one line without control characters, at most 200 characters. +func cleanSummary(s string) string { + s = strings.Map(func(r rune) rune { + if unicode.IsControl(r) { + return -1 + } + return r + }, strings.Join(strings.Fields(s), " ")) + if r := []rune(s); len(r) > 200 { + s = string(r[:200]) + } + return s } // Close closes the database.