From 2abd435494a69d094e30e9f58a89a62ff56b8d1d Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 12:37:52 +0100 Subject: [PATCH 1/7] feat(store): record posts and count their deliveries - posts table: one row per post queued to a list (message id, subject, time, recipients, sent, failed, dropped), deleted with the list and forgotten after 400 days when a new post is recorded - EnqueuePost and EnqueueFromHeld record the post in the same transaction and stamp every delivery with its id; a refused duplicate records nothing; other mail keeps post_id 0 - CompleteOutbox counts sent, the new DropOutbox counts dropped and FailOutbox counts failed, each in the transaction that changes the delivery; a delivery that already failed is never counted twice - outbox priority column; DueOutbox orders by priority, next_at, id - ListPosts newest first (default 50, at most 200) with Pending derived - additive migration of the outbox columns, tested on an old database --- internal/store/outbox.go | 92 ++++++-- internal/store/posts.go | 78 +++++++ internal/store/posts_test.go | 421 +++++++++++++++++++++++++++++++++++ internal/store/store.go | 51 +++-- 4 files changed, 608 insertions(+), 34 deletions(-) create mode 100644 internal/store/posts.go create mode 100644 internal/store/posts_test.go diff --git a/internal/store/outbox.go b/internal/store/outbox.go index c72abea..a0aea3b 100644 --- a/internal/store/outbox.go +++ b/internal/store/outbox.go @@ -12,6 +12,7 @@ type NewOutbound struct { Rcpt string MemberID int64 // 0 for system mail Prefix []byte // per-recipient headers, sent before the shared message + Priority int // lower goes first among due deliveries: 0 for discussion and system mail, 1 for a newsletter } // OutboxItem is a queued delivery to one recipient. @@ -28,7 +29,8 @@ type OutboxItem struct { CreatedAt time.Time // Failed deliveries are never retried (the relay refused them for good, or they are too old). // They stay listed for a week so the admin can see what was lost and why. - Failed bool + Failed bool + Priority int } // OutboxStats summarises the queue. @@ -52,7 +54,7 @@ func (s *Store) Enqueue(msg []byte, items []NewOutbound) error { return err } defer tx.Rollback() - if err := enqueueTx(tx, s.Now().Unix(), msg, items); err != nil { + if err := enqueueTx(tx, s.Now().Unix(), 0, msg, items); err != nil { return err } return tx.Commit() @@ -65,6 +67,8 @@ const postMemory = time.Hour // in the same transaction, so a crash never leaves one without the other. If the list already // queued the same postID within postMemory it queues nothing and returns ErrDuplicate: the MTA // delivers the post again when our reply to the first delivery was lost. +// It also records the post (a Post, with its counters) and stamps every delivery with its id; +// a refused duplicate records nothing. func (s *Store) EnqueuePost(listID int64, postID string, msg []byte, items []NewOutbound) error { now := s.Now() tx, err := s.db.Begin() @@ -81,8 +85,12 @@ func (s *Store) EnqueuePost(listID int64, postID string, msg []byte, items []New } return err } + post, err := recordPostTx(tx, now, listID, msg, len(items)) + if err != nil { + return err + } if len(items) > 0 { - if err := enqueueTx(tx, now.Unix(), msg, items); err != nil { + if err := enqueueTx(tx, now.Unix(), post, msg, items); err != nil { return err } } @@ -93,7 +101,7 @@ func (s *Store) EnqueuePost(listID int64, postID string, msg []byte, items []New // its deliveries in one transaction. Of several simultaneous approvals only one succeeds, the // others get ErrNotFound, so a message is never delivered twice. With no recipients the message // is still released. An approval always delivers, even a post already seen, and records postID -// as EnqueuePost does, so a later copy of the same post is refused. +// as EnqueuePost does, so a later copy of the same post is refused. It records the post as EnqueuePost does. func (s *Store) EnqueueFromHeld(heldID int64, postID string, msg []byte, items []NewOutbound) error { tx, err := s.db.Begin() if err != nil { @@ -113,15 +121,20 @@ func (s *Store) EnqueueFromHeld(heldID int64, postID string, msg []byte, items [ ON CONFLICT(list_id, post_id) DO UPDATE SET created_at=excluded.created_at`, listID, postID, s.Now().Unix()); err != nil { return err } + post, err := recordPostTx(tx, s.Now(), listID, msg, len(items)) + if err != nil { + return err + } if len(items) > 0 { - if err := enqueueTx(tx, s.Now().Unix(), msg, items); err != nil { + if err := enqueueTx(tx, s.Now().Unix(), post, msg, items); err != nil { return err } } return tx.Commit() } -func enqueueTx(tx *sql.Tx, now int64, msg []byte, items []NewOutbound) error { +// enqueueTx stores msg and its deliveries; postID is the posts row they count for, 0 for other mail. +func enqueueTx(tx *sql.Tx, now, postID int64, msg []byte, items []NewOutbound) error { res, err := tx.Exec(`INSERT INTO outbox_msgs(data) VALUES(?)`, msg) if err != nil { return err @@ -132,20 +145,20 @@ func enqueueTx(tx *sql.Tx, now int64, msg []byte, items []NewOutbound) error { if prefix == nil { prefix = []byte{} } - if _, err := tx.Exec(`INSERT INTO outbox(msg_id, envelope_from, rcpt, member_id, prefix, next_at, created_at) - VALUES(?,?,?,?,?,?,?)`, msgID, it.From, it.Rcpt, it.MemberID, prefix, now, now); err != nil { + if _, err := tx.Exec(`INSERT INTO outbox(msg_id, envelope_from, rcpt, member_id, prefix, next_at, created_at, post_id, priority) + VALUES(?,?,?,?,?,?,?,?,?)`, msgID, it.From, it.Rcpt, it.MemberID, prefix, now, now, postID, it.Priority); err != nil { return err } } return nil } -const outboxCols = `id, msg_id, envelope_from, rcpt, member_id, prefix, attempts, next_at, last_error, created_at, failed` +const outboxCols = `id, msg_id, envelope_from, rcpt, member_id, prefix, attempts, next_at, last_error, created_at, failed, priority` func scanOutbox(row interface{ Scan(...any) error }) (OutboxItem, error) { var it OutboxItem var next, created, failed int64 - err := row.Scan(&it.ID, &it.MsgID, &it.From, &it.Rcpt, &it.MemberID, &it.Prefix, &it.Attempts, &next, &it.LastError, &created, &failed) + err := row.Scan(&it.ID, &it.MsgID, &it.From, &it.Rcpt, &it.MemberID, &it.Prefix, &it.Attempts, &next, &it.LastError, &created, &failed, &it.Priority) it.NextAt, it.CreatedAt, it.Failed = time.Unix(next, 0), time.Unix(created, 0), failed != 0 return it, err } @@ -167,9 +180,10 @@ func (s *Store) queryOutbox(q string, args ...any) ([]OutboxItem, error) { return out, rows.Err() } -// DueOutbox returns up to limit deliveries whose time has come, longest waiting first. +// DueOutbox returns up to limit deliveries whose time has come: lowest priority number first (so a +// big newsletter never delays discussion mail), then longest waiting first. func (s *Store) DueOutbox(limit int) ([]OutboxItem, error) { - return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox WHERE failed=0 AND next_at<=? ORDER BY next_at, id LIMIT ?`, + return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox WHERE failed=0 AND next_at<=? ORDER BY priority, next_at, id LIMIT ?`, s.Now().Unix(), limit) } @@ -188,16 +202,26 @@ func (s *Store) OutboxData(msgID int64) ([]byte, error) { return b, err } -// CompleteOutbox removes a delivery (sent, dropped or discarded) and the shared -// message once no recipient needs it. -func (s *Store) CompleteOutbox(id int64) error { +// CompleteOutbox removes a delivery the relay accepted, and the shared message once no recipient +// needs it. A delivery of a post counts as sent, in the same transaction. +func (s *Store) CompleteOutbox(id int64) error { return s.removeOutbox(id, "sent") } + +// DropOutbox removes a delivery that will never be made (the mailbox does not exist, or the admin +// discarded it), like CompleteOutbox. A delivery of a post counts as dropped, unless it had +// already failed: it is counted as failed already. +func (s *Store) DropOutbox(id int64) error { return s.removeOutbox(id, "dropped") } + +// removeOutbox deletes a delivery and adds one to counter (a column of posts, never user input) +// of its post, when it has one and it had not failed. +func (s *Store) removeOutbox(id int64, counter string) error { tx, err := s.db.Begin() if err != nil { return err } defer tx.Rollback() - var msgID int64 - err = tx.QueryRow(`SELECT msg_id FROM outbox WHERE id=?`, id).Scan(&msgID) + var msgID, postID int64 + var failed bool + err = tx.QueryRow(`SELECT msg_id, post_id, failed FROM outbox WHERE id=?`, id).Scan(&msgID, &postID, &failed) if errors.Is(err, sql.ErrNoRows) { return ErrNotFound } @@ -210,6 +234,11 @@ func (s *Store) CompleteOutbox(id int64) error { if _, err := tx.Exec(`DELETE FROM outbox_msgs WHERE id=? AND NOT EXISTS (SELECT 1 FROM outbox WHERE msg_id=?)`, msgID, msgID); err != nil { return err } + if postID != 0 && !failed { + if _, err := tx.Exec(`UPDATE posts SET `+counter+`=`+counter+`+1 WHERE id=?`, postID); err != nil { + return err + } + } return tx.Commit() } @@ -232,10 +261,33 @@ func (s *Store) OutboxStats() (OutboxStats, error) { } // FailOutbox gives up on a delivery for good, keeping it listed with the reason. Its time is the -// moment of the failure, from which PurgeFailedOutbox counts. +// moment of the failure, from which PurgeFailedOutbox counts. A delivery of a post counts as failed, +// in the same transaction. func (s *Store) FailOutbox(id int64, reason string) error { - return affected(s.db.Exec(`UPDATE outbox SET failed=1, attempts=attempts+1, last_error=?, next_at=? WHERE id=?`, - reason, s.Now().Unix(), id)) + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + var postID int64 + var failed bool + err = tx.QueryRow(`SELECT post_id, failed FROM outbox WHERE id=?`, id).Scan(&postID, &failed) + if errors.Is(err, sql.ErrNoRows) { + return ErrNotFound + } + if err != nil { + return err + } + if _, err := tx.Exec(`UPDATE outbox SET failed=1, attempts=attempts+1, last_error=?, next_at=? WHERE id=?`, + reason, s.Now().Unix(), id); err != nil { + return err + } + if postID != 0 && !failed { + if _, err := tx.Exec(`UPDATE posts SET failed=failed+1 WHERE id=?`, postID); err != nil { + return err + } + } + return tx.Commit() } // PurgeFailedOutbox forgets failed deliveries older than a week, and the stored messages nobody diff --git a/internal/store/posts.go b/internal/store/posts.go new file mode 100644 index 0000000..e9f3323 --- /dev/null +++ b/internal/store/posts.go @@ -0,0 +1,78 @@ +package store + +import ( + "bytes" + "database/sql" + "net/mail" + "time" +) + +// Post is a post queued to a list, with the outcome of its deliveries so far. +type Post struct { + ID int64 + ListID int64 + MessageID string // the Message-ID header, at most 200 characters + Subject string // decoded, at most 200 characters + CreatedAt time.Time + Recipients int // deliveries queued + Sent int // accepted by the relay + Failed int // given up for good + Dropped int // the mailbox does not exist, or discarded by the admin + Pending int // still in the queue: Recipients - Sent - Failed - Dropped +} + +// postTTL is how long a post and its counters are kept. +const postTTL = 400 * 24 * time.Hour + +// Limits of ListPosts. +const ( + defaultPostsLimit = 50 + maxPostsLimit = 200 +) + +// recordPostTx stores the posts row of a post queued to a list and returns its id. It forgets +// posts older than postTTL on the way, so the table never grows without bound. +func recordPostTx(tx *sql.Tx, now time.Time, listID int64, msg []byte, recipients int) (int64, error) { + if _, err := tx.Exec(`DELETE FROM posts WHERE created_at maxPostsLimit { + limit = maxPostsLimit + } + rows, err := s.db.Query(`SELECT id, list_id, message_id, subject, created_at, recipients, sent, failed, dropped + FROM posts WHERE list_id=? ORDER BY created_at DESC, id DESC LIMIT ?`, listID, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Post + for rows.Next() { + var p Post + var created int64 + if err := rows.Scan(&p.ID, &p.ListID, &p.MessageID, &p.Subject, &created, &p.Recipients, &p.Sent, &p.Failed, &p.Dropped); err != nil { + return nil, err + } + p.CreatedAt = time.Unix(created, 0) + p.Pending = p.Recipients - p.Sent - p.Failed - p.Dropped + out = append(out, p) + } + return out, rows.Err() +} diff --git a/internal/store/posts_test.go b/internal/store/posts_test.go new file mode 100644 index 0000000..1c9bce1 --- /dev/null +++ b/internal/store/posts_test.go @@ -0,0 +1,421 @@ +package store + +import ( + "errors" + "strings" + "testing" + "time" +) + +const postMsg = "Message-ID: \r\nSubject: =?utf-8?q?Caff=C3=A8?= news\x07\r\n\r\nbody" + +func onlyPost(t *testing.T, s *Store, listID int64) Post { + t.Helper() + posts, err := s.ListPosts(listID, 0) + if err != nil || len(posts) != 1 { + t.Fatalf("posts: %+v %v", posts, err) + } + return posts[0] +} + +func outboxPostIDs(t *testing.T, s *Store) []int64 { + t.Helper() + rows, err := s.db.Query(`SELECT post_id FROM outbox ORDER BY id`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var ids []int64 + for rows.Next() { + var id int64 + rows.Scan(&id) + ids = append(ids, id) + } + return ids +} + +func TestEnqueuePostRecordsThePost(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + l, _ := s.CreateList("dev", "example.org", "Dev") + if err := s.EnqueuePost(l.ID, "mid:", []byte(postMsg), items(3)); err != nil { + t.Fatal(err) + } + p := onlyPost(t, s, l.ID) + if p.ID == 0 || p.ListID != l.ID || p.MessageID != "" || p.Subject != "Caffè news" || !p.CreatedAt.Equal(now) || + p.Recipients != 3 || p.Sent != 0 || p.Failed != 0 || p.Dropped != 0 || p.Pending != 3 { + t.Fatalf("post: %+v", p) + } + for _, id := range outboxPostIDs(t, s) { + if id != p.ID { + t.Fatalf("every delivery carries the post id: %d != %d", id, p.ID) + } + } +} + +func TestEnqueuePostRecordsAPostRowWithNoRecipients(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + if err := s.EnqueuePost(l.ID, "mid:", []byte(postMsg), nil); err != nil { + t.Fatal(err) + } + if p := onlyPost(t, s, l.ID); p.Recipients != 0 || p.Pending != 0 { + t.Fatalf("post: %+v", p) + } +} + +func TestARefusedDuplicateRecordsNoPost(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.EnqueuePost(l.ID, "mid:", []byte(postMsg), items(1)) + if err := s.EnqueuePost(l.ID, "mid:", []byte(postMsg), items(1)); !errors.Is(err, ErrDuplicate) { + t.Fatalf("want ErrDuplicate, got %v", err) + } + onlyPost(t, s, l.ID) +} + +func TestAFailedQueueWriteRecordsNoPost(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + if _, err := s.db.Exec(`CREATE TRIGGER boom BEFORE INSERT ON outbox BEGIN SELECT RAISE(ABORT, 'boom'); END`); err != nil { + t.Fatal(err) + } + if err := s.EnqueuePost(l.ID, "mid:", []byte(postMsg), items(2)); err == nil { + t.Fatal("want the queue error") + } + hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r") + if err := s.EnqueueFromHeld(hid, "mid:", []byte(postMsg), items(2)); err == nil { + t.Fatal("want the queue error") + } + if posts, _ := s.ListPosts(l.ID, 0); len(posts) != 0 { + t.Fatalf("posts: %+v", posts) + } +} + +func TestReleasingAHeldPostRecordsItOnce(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r") + if err := s.EnqueueFromHeld(hid, "mid:", []byte(postMsg), items(2)); err != nil { + t.Fatal(err) + } + s.EnqueueFromHeld(hid, "mid:", []byte(postMsg), items(2)) // a second approval: ErrNotFound + p := onlyPost(t, s, l.ID) + if p.Recipients != 2 || p.Subject != "Caffè news" { + t.Fatalf("post: %+v", p) + } + for _, id := range outboxPostIDs(t, s) { + if id != p.ID { + t.Fatalf("delivery post id %d != %d", id, p.ID) + } + } + hid2, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r") + if err := s.EnqueueFromHeld(hid2, "mid:", []byte(postMsg), nil); err != nil { + t.Fatal(err) + } + if posts, _ := s.ListPosts(l.ID, 0); len(posts) != 2 || posts[0].Recipients != 0 { + t.Fatalf("a released post with no recipients has a row too: %+v", posts) + } +} + +func TestOtherMailIsNotAPost(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + if err := s.Enqueue([]byte(postMsg), items(2)); err != nil { + t.Fatal(err) + } + if posts, _ := s.ListPosts(l.ID, 0); len(posts) != 0 { + t.Fatalf("posts: %+v", posts) + } + for _, id := range outboxPostIDs(t, s) { + if id != 0 { + t.Fatalf("post id %d", id) + } + } +} + +func TestPostSubjectIsCutAndClean(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + msg := "Message-ID: \r\nSubject: " + strings.Repeat("é", 300) + "\r\n\r\nb" + s.EnqueuePost(l.ID, "mid:x", []byte(msg), items(1)) + p := onlyPost(t, s, l.ID) + if p.Subject != strings.Repeat("é", 200) || p.MessageID != "" { + t.Fatalf("post: %+v", p) + } +} + +func TestCountersFollowTheDeliveries(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + l, _ := s.CreateList("dev", "example.org", "Dev") + s.EnqueuePost(l.ID, "mid:", []byte(postMsg), items(6)) + due, _ := s.DueOutbox(10) + + if err := s.RetryOutbox(due[0].ID, now.Add(time.Minute), "down"); err != nil { + t.Fatal(err) + } + if p := onlyPost(t, s, l.ID); p.Sent+p.Failed+p.Dropped != 0 || p.Pending != 6 { + t.Fatalf("a retry changes nothing: %+v", p) + } + if err := s.CompleteOutbox(due[1].ID); err != nil { + t.Fatal(err) + } + if err := s.FailOutbox(due[2].ID, "refused"); err != nil { + t.Fatal(err) + } + if err := s.DropOutbox(due[3].ID); err != nil { + t.Fatal(err) + } + if err := s.DropOutbox(due[4].ID); err != nil { + t.Fatal(err) + } + p := onlyPost(t, s, l.ID) + if p.Sent != 1 || p.Failed != 1 || p.Dropped != 2 || p.Pending != 2 { + t.Fatalf("post: %+v", p) + } + // Discarding a failed delivery: it is already counted as failed. + if err := s.DropOutbox(due[2].ID); err != nil { + t.Fatal(err) + } + if err := s.CompleteOutbox(due[2].ID); !errors.Is(err, ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } + if err := s.DropOutbox(due[2].ID); !errors.Is(err, ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } + if p2 := onlyPost(t, s, l.ID); p2 != p { + t.Fatalf("discarding a failed delivery changes nothing: %+v", p2) + } +} + +func TestCompletingAFailedDeliveryDoesNotCountItAsSent(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.EnqueuePost(l.ID, "mid:", []byte(postMsg), items(1)) + due, _ := s.DueOutbox(10) + s.FailOutbox(due[0].ID, "refused") + if err := s.CompleteOutbox(due[0].ID); err != nil { + t.Fatal(err) + } + if p := onlyPost(t, s, l.ID); p.Sent != 0 || p.Failed != 1 || p.Pending != 0 { + t.Fatalf("post: %+v", p) + } +} + +func TestDropOutboxRemovesTheDeliveryAndTheBody(t *testing.T) { + s := newTestStore(t) + s.Enqueue([]byte("body"), items(1)) + due, _ := s.DueOutbox(10) + if err := s.DropOutbox(due[0].ID); err != nil { + t.Fatal(err) + } + if st, _ := s.OutboxStats(); st.Pending != 0 || st.Failed != 0 { + t.Fatalf("stats: %+v", st) + } + if _, err := s.OutboxData(due[0].MsgID); !errors.Is(err, ErrNotFound) { + t.Fatalf("body: %v", err) + } +} + +func TestPurgingOldFailuresKeepsTheCounters(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + l, _ := s.CreateList("dev", "example.org", "Dev") + s.EnqueuePost(l.ID, "mid:", []byte(postMsg), items(2)) + due, _ := s.DueOutbox(10) + s.FailOutbox(due[0].ID, "refused") + now = now.Add(8 * 24 * time.Hour) + if n, err := s.PurgeFailedOutbox(); err != nil || n != 1 { + t.Fatalf("purged %d %v", n, err) + } + if p := onlyPost(t, s, l.ID); p.Failed != 1 || p.Pending != 1 { + t.Fatalf("post: %+v", p) + } +} + +func TestListPostsNewestFirstWithALimit(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + dev, _ := s.CreateList("dev", "example.org", "Dev") + ops, _ := s.CreateList("ops", "example.org", "Ops") + for i := 0; i < 210; i++ { + now = now.Add(time.Second) + if err := s.EnqueuePost(dev.ID, "mid:"+string(rune(0x100+i)), []byte(postMsg), nil); err != nil { + t.Fatal(err) + } + } + s.EnqueuePost(ops.ID, "mid:ops", []byte(postMsg), items(1)) + all, _ := s.ListPosts(dev.ID, 1000) + if len(all) != 200 { + t.Fatalf("capped at 200: %d", len(all)) + } + for i := 1; i < len(all); i++ { + if !all[i-1].CreatedAt.After(all[i].CreatedAt) { + t.Fatalf("not newest first at %d", i) + } + } + if def, _ := s.ListPosts(dev.ID, 0); len(def) != 50 || def[0].ID != all[0].ID { + t.Fatalf("default 50: %d", len(def)) + } + if neg, _ := s.ListPosts(dev.ID, -1); len(neg) != 50 { + t.Fatalf("default 50: %d", len(neg)) + } + if some, _ := s.ListPosts(dev.ID, 7); len(some) != 7 { + t.Fatalf("limit 7: %d", len(some)) + } + if exact, _ := s.ListPosts(dev.ID, 200); len(exact) != 200 { + t.Fatalf("limit 200: %d", len(exact)) + } + if o, _ := s.ListPosts(ops.ID, 0); len(o) != 1 || o[0].Recipients != 1 { + t.Fatalf("other list: %+v", o) + } +} + +func TestListPostsBreaksTiesByID(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.Now = func() time.Time { return time.Unix(1_000_000, 0) } + s.EnqueuePost(l.ID, "mid:1", []byte(postMsg), nil) + s.EnqueuePost(l.ID, "mid:2", []byte(postMsg), nil) + posts, _ := s.ListPosts(l.ID, 0) + if len(posts) != 2 || posts[0].ID < posts[1].ID { + t.Fatalf("the later post first: %+v", posts) + } +} + +func TestOldPostsArePurgedWhenANewOneIsRecorded(t *testing.T) { + s := newTestStore(t) + now := time.Unix(100_000_000, 0) + s.Now = func() time.Time { return now } + dev, _ := s.CreateList("dev", "example.org", "Dev") + ops, _ := s.CreateList("ops", "example.org", "Ops") + s.EnqueuePost(dev.ID, "mid:old", []byte(postMsg), nil) + now = now.Add(time.Second) + s.EnqueuePost(dev.ID, "mid:kept", []byte(postMsg), nil) + kept := now + now = now.Add(400 * 24 * time.Hour) // the first is now older than 400 days, the second exactly 400 days old + hid, _ := s.HoldMessage(ops.ID, "a@x.com", []byte("x"), "r") + if err := s.EnqueueFromHeld(hid, "mid:new", []byte(postMsg), nil); err != nil { + t.Fatal(err) + } + posts, _ := s.ListPosts(dev.ID, 0) + if len(posts) != 1 || !posts[0].CreatedAt.Equal(kept) { + t.Fatalf("posts: %+v", posts) + } + now = now.Add(400 * 24 * time.Hour) + s.EnqueuePost(ops.ID, "mid:newer", []byte(postMsg), nil) + if posts, _ := s.ListPosts(dev.ID, 0); len(posts) != 0 { + t.Fatalf("EnqueuePost purges too: %+v", posts) + } +} + +func TestDeletingAListDeletesItsPosts(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.EnqueuePost(l.ID, "mid:", []byte(postMsg), nil) + if err := s.DeleteList(l.ID); err != nil { + t.Fatal(err) + } + var n int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM posts`).Scan(&n); err != nil || n != 0 { + t.Fatalf("posts left: %d %v", n, err) + } +} + +func TestMigrationAddsPostAndPriorityColumns(t *testing.T) { + path := t.TempDir() + "/old.db" + s, _ := Open(path) + s.Enqueue([]byte("x"), items(1)) + for _, q := range []string{`DROP INDEX IF EXISTS outbox_due_prio`, `ALTER TABLE outbox DROP COLUMN post_id`, + `ALTER TABLE outbox DROP COLUMN priority`, `DROP TABLE posts`} { + if _, err := s.db.Exec(q); err != nil { + t.Fatalf("%s: %v", q, err) + } + } + s.Close() + s2, err := Open(path) + if err != nil { + t.Fatalf("migrate: %v", err) + } + defer s2.Close() + due, _ := s2.DueOutbox(10) + if len(due) != 1 { + t.Fatalf("an existing delivery stays waiting: %+v", due) + } + if ids := outboxPostIDs(t, s2); len(ids) != 1 || ids[0] != 0 { + t.Fatalf("post ids: %v", ids) + } + l, _ := s2.CreateList("dev", "example.org", "Dev") + if err := s2.EnqueuePost(l.ID, "mid:", []byte(postMsg), items(1)); err != nil { + t.Fatal(err) + } + onlyPost(t, s2, l.ID) +} + +func TestDueOutboxSendsLowerPriorityNumbersFirst(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + news := items(3) + for i := range news { + news[i].Priority = 1 + } + s.Enqueue([]byte("newsletter"), news) + now = now.Add(time.Minute) + s.Enqueue([]byte("discussion"), items(2)) + due, _ := s.DueOutbox(10) + if len(due) != 5 { + t.Fatalf("due: %d", len(due)) + } + for i, d := range due { + if d.Priority != map[bool]int{true: 0, false: 1}[i < 2] { + t.Fatalf("priority 0 first: %+v", due) + } + } + for i := 1; i < len(due); i++ { + if due[i].Priority == due[i-1].Priority && due[i].ID < due[i-1].ID { + t.Fatalf("FIFO within a priority: %+v", due) + } + } + if got, _ := s.DueOutbox(2); got[0].Priority != 0 || got[1].Priority != 0 { + t.Fatalf("the limit takes the first in priority: %+v", got) + } + // A priority-0 delivery that is not due yet does not jump ahead. + s.RetryOutbox(due[0].ID, now.Add(time.Hour), "down") + s.RetryOutbox(due[1].ID, now.Add(time.Hour), "down") + if got, _ := s.DueOutbox(10); len(got) != 3 || got[0].Priority != 1 { + t.Fatalf("due: %+v", got) + } +} + +func TestWithinAPriorityTheLongestWaitingGoesFirst(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + s.Enqueue([]byte("body"), items(3)) + due, _ := s.DueOutbox(10) + s.RetryOutbox(due[2].ID, now.Add(-time.Hour), "e") + got, _ := s.DueOutbox(10) + if got[0].ID != due[2].ID || got[1].ID != due[0].ID || got[2].ID != due[1].ID { + t.Fatalf("order: %+v", got) + } +} + +func TestFailingAFailedDeliveryAgainCountsItOnce(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.EnqueuePost(l.ID, "mid:", []byte(postMsg), items(1)) + due, _ := s.DueOutbox(10) + s.FailOutbox(due[0].ID, "refused") + if err := s.FailOutbox(due[0].ID, "refused again"); err != nil { + t.Fatal(err) + } + if p := onlyPost(t, s, l.ID); p.Failed != 1 || p.Pending != 0 { + t.Fatalf("post: %+v", p) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 705bb1a..a046d4a 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -212,9 +212,24 @@ CREATE TABLE IF NOT EXISTS outbox ( next_at INTEGER NOT NULL, last_error TEXT NOT NULL DEFAULT '', created_at INTEGER NOT NULL, - failed INTEGER NOT NULL DEFAULT 0 + failed INTEGER NOT NULL DEFAULT 0, + post_id INTEGER NOT NULL DEFAULT 0, + priority INTEGER NOT NULL DEFAULT 0 ); CREATE INDEX IF NOT EXISTS outbox_due ON outbox(next_at, id); +CREATE INDEX IF NOT EXISTS outbox_due_prio ON outbox(priority, next_at, id); +CREATE TABLE IF NOT EXISTS posts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE, + message_id TEXT NOT NULL DEFAULT '', + subject TEXT NOT NULL DEFAULT '', + created_at INTEGER NOT NULL, + recipients INTEGER NOT NULL DEFAULT 0, + sent INTEGER NOT NULL DEFAULT 0, + failed INTEGER NOT NULL DEFAULT 0, + dropped INTEGER NOT NULL DEFAULT 0 +); +CREATE INDEX IF NOT EXISTS posts_list ON posts(list_id, created_at, id); CREATE TABLE IF NOT EXISTS delivered ( list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE, post_id TEXT NOT NULL, @@ -302,6 +317,12 @@ func migrate(db *sql.DB) error { return err } } + if has, _ := hasColumn(db, "outbox", "post_id"); !has { + if _, err := db.Exec(`ALTER TABLE outbox ADD COLUMN post_id INTEGER NOT NULL DEFAULT 0; + ALTER TABLE outbox ADD COLUMN priority INTEGER NOT NULL DEFAULT 0;`); err != nil { + return err + } + } } if ok, err := hasColumn(db, "held", "list_id"); err != nil { return err @@ -390,23 +411,25 @@ func summarize(data []byte) (subject, from string) { if err != nil { return "", "" } - clean := func(s string) string { - s = strings.Map(func(r rune) rune { - if unicode.IsControl(r) { - return -1 - } - return r - }, strings.Join(strings.Fields(s), " ")) - if r := []rune(s); len(r) > 200 { - s = string(r[:200]) - } - return s - } raw := m.Header.Get("Subject") if dec, err := new(mime.WordDecoder).DecodeHeader(raw); err == nil { raw = dec } - return clean(raw), clean(m.Header.Get("From")) + return cleanSummary(raw), cleanSummary(m.Header.Get("From")) +} + +// cleanSummary makes a header value fit for a listing: one line without control characters, at most 200 characters. +func cleanSummary(s string) string { + s = strings.Map(func(r rune) rune { + if unicode.IsControl(r) { + return -1 + } + return r + }, strings.Join(strings.Fields(s), " ")) + if r := []rune(s); len(r) > 200 { + s = string(r[:200]) + } + return s } // Close closes the database. From 51e2bf683e634185cdc18032f7b26dbc7897d095 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 12:38:32 +0100 Subject: [PATCH 2/7] feat(queue): count sent, failed and dropped deliveries of a post - a delivery the relay accepted completes as sent, one given up as failed, one to a missing mailbox (or with a missing body) is dropped through store.DropOutbox; a retry changes nothing - tests: each outcome of a post, giving up after MaxAge, a missing body, and discussion mail sent before a bigger newsletter queued earlier --- internal/queue/posts_test.go | 114 +++++++++++++++++++++++++++++++++++ internal/queue/queue.go | 11 +++- 2 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 internal/queue/posts_test.go diff --git a/internal/queue/posts_test.go b/internal/queue/posts_test.go new file mode 100644 index 0000000..3736527 --- /dev/null +++ b/internal/queue/posts_test.go @@ -0,0 +1,114 @@ +package queue + +import ( + "context" + "errors" + "testing" + + "github.com/emersion/go-smtp" + + "codeberg.org/squidd/parsimony/internal/outbound" + "codeberg.org/squidd/parsimony/internal/store" +) + +// post queues a post of n recipients (a@x.com, b@x.com...) to a new list and returns the list. +func (e *env) post(t *testing.T, n int) store.List { + t.Helper() + l, err := e.s.CreateList("dev", "example.org", "Dev") + if err != nil { + t.Fatal(err) + } + var items []store.NewOutbound + for i := 0; i < n; i++ { + items = append(items, store.NewOutbound{From: "bounce@x.org", Rcpt: string(rune('a'+i)) + "@x.com"}) + } + if err := e.s.EnqueuePost(l.ID, "mid:", []byte("Message-ID: \r\nSubject: hi\r\n\r\nbody\r\n"), items); err != nil { + t.Fatal(err) + } + return l +} + +func (e *env) onlyPost(t *testing.T, l store.List) store.Post { + t.Helper() + posts, err := e.s.ListPosts(l.ID, 0) + if err != nil || len(posts) != 1 { + t.Fatalf("posts: %+v %v", posts, err) + } + return posts[0] +} + +func TestTheWorkerCountsEachOutcomeOfAPost(t *testing.T) { + e := setup(t) + l := e.post(t, 4) + e.snd.fn = func(_ context.Context, _ string, to []string) error { + switch to[0] { + case "b@x.com": + return errors.New("connection refused") + case "c@x.com": + return &smtp.SMTPError{Code: 554, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "policy"} + case "d@x.com": + return outbound.AtRcpt(&smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"}) + } + return nil + } + st, err := e.w.RunOnce(context.Background()) + if err != nil || st.Sent != 1 || st.Retried != 1 || st.Failed != 1 || st.Dropped != 1 { + t.Fatalf("run: %+v %v", st, err) + } + p := e.onlyPost(t, l) + if p.Recipients != 4 || p.Sent != 1 || p.Failed != 1 || p.Dropped != 1 || p.Pending != 1 { + t.Fatalf("post: %+v", p) + } + // The retried delivery goes out later. + e.snd.fn = nil + e.clk.set(t0.Add(Backoff(1))) + e.w.RunOnce(context.Background()) + if p := e.onlyPost(t, l); p.Sent != 2 || p.Pending != 0 { + t.Fatalf("post: %+v", p) + } +} + +func TestGivingUpAfterMaxAgeCountsAsFailed(t *testing.T) { + e := setup(t) + l := e.post(t, 1) + e.snd.fn = func(context.Context, string, []string) error { return errors.New("down") } + e.clk.set(t0.Add(MaxAge + 1)) + if st, _ := e.w.RunOnce(context.Background()); st.Failed != 1 { + t.Fatalf("stats: %+v", st) + } + if p := e.onlyPost(t, l); p.Failed != 1 || p.Pending != 0 { + t.Fatalf("post: %+v", p) + } +} + +func TestAMissingBodyCountsAsDropped(t *testing.T) { + e := setup(t) + e.w.readBody = func(int64) ([]byte, error) { return nil, store.ErrNotFound } + l := e.post(t, 2) + e.w.RunOnce(context.Background()) + if p := e.onlyPost(t, l); p.Dropped != 2 || p.Sent != 0 || p.Pending != 0 { + t.Fatalf("post: %+v", p) + } +} + +func TestDiscussionMailIsSentBeforeABigNewsletterQueuedEarlier(t *testing.T) { + e := setup(t) + e.w.Concurrency = 1 + var news []store.NewOutbound + for i := 0; i < 120; i++ { + news = append(news, store.NewOutbound{From: "b@x.org", Rcpt: "reader@x.com", Priority: 1}) + } + if err := e.s.Enqueue([]byte("Subject: newsletter\r\n\r\nx"), news); err != nil { + t.Fatal(err) + } + e.clk.set(t0.Add(1)) + e.s.Enqueue([]byte("Subject: talk\r\n\r\nx"), []store.NewOutbound{ + {From: "b@x.org", Rcpt: "first@x.com"}, {From: "b@x.org", Rcpt: "second@x.com"}, + }) + if _, err := e.w.RunOnce(context.Background()); err != nil { + t.Fatal(err) + } + if e.snd.sent[0].To[0] != "first@x.com" || e.snd.sent[1].To[0] != "second@x.com" || e.snd.sent[2].To[0] != "reader@x.com" { + t.Fatalf("order: %v %v %v", e.snd.sent[0].To, e.snd.sent[1].To, e.snd.sent[2].To) + } +} diff --git a/internal/queue/queue.go b/internal/queue/queue.go index 9976a9c..77f8141 100644 --- a/internal/queue/queue.go +++ b/internal/queue/queue.go @@ -219,7 +219,7 @@ func (w *Worker) RunOnce(ctx context.Context) (Stats, error) { func (w *Worker) deliver(ctx context.Context, cache *bodies, it store.OutboxItem) Stats { data, err := cache.get(it.MsgID) if errors.Is(err, store.ErrNotFound) { - w.complete(it) + w.drop(it) return Stats{Dropped: 1} } if err != nil { @@ -259,7 +259,7 @@ func (w *Worker) deliver(ctx context.Context, cache *bodies, it store.OutboxItem } } log.Printf("queue: drop %s: %v", it.Rcpt, err) - w.complete(it) + w.drop(it) return out case outbound.IsPermanent(err): // The relay refused the message itself (policy, size, line length...): nobody gets it, so @@ -292,3 +292,10 @@ func (w *Worker) complete(it store.OutboxItem) { log.Printf("queue: remove %d: %v", it.ID, err) } } + +// drop removes a delivery that will never be made; a delivery of a post counts as dropped. +func (w *Worker) drop(it store.OutboxItem) { + if err := w.Store.DropOutbox(it.ID); err != nil && !errors.Is(err, store.ErrNotFound) { + log.Printf("queue: drop %d: %v", it.ID, err) + } +} From 0dc9c03bae6d5301c77fc6f1b5274d6fcce5e2e5 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 12:39:34 +0100 Subject: [PATCH 3/7] feat(router): send newsletter deliveries with priority 1 - Deliver and Release (through outbound) queue the deliveries of a list whose post policy is publishers with priority 1, everything else 0; confirmation mail stays 0 - tests for Deliver, Release, confirmation mail and the recorded post --- internal/router/priority_test.go | 127 +++++++++++++++++++++++++++++++ internal/router/router.go | 9 ++- 2 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 internal/router/priority_test.go diff --git a/internal/router/priority_test.go b/internal/router/priority_test.go new file mode 100644 index 0000000..fd9ef41 --- /dev/null +++ b/internal/router/priority_test.go @@ -0,0 +1,127 @@ +package router + +import ( + "database/sql" + "testing" + + "codeberg.org/squidd/parsimony/internal/store" +) + +// newsletterEnv is newEnv on a database file whose list has the publishers post policy, set +// directly in the database. +func newsletterEnv(t *testing.T) *env { + t.Helper() + path := t.TempDir() + "/p.db" + s, err := store.Open(path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.AddMember(l.ID, "alice@example.com", "Alice") + s.AddMember(l.ID, "bob@example.com", "Bob") + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`UPDATE lists SET post_policy=? WHERE id=?`, store.PostPublishers, l.ID); err != nil { + t.Fatal(err) + } + l, _ = s.GetList(l.ID) + return &env{r: New(s), s: s, list: l} +} + +func duePriorities(t *testing.T, s *store.Store) []int { + t.Helper() + due, err := s.DueOutbox(100) + if err != nil { + t.Fatal(err) + } + var out []int + for _, d := range due { + out = append(out, d.Priority) + } + return out +} + +func allEqual(ps []int, want int, n int) bool { + if len(ps) != n { + return false + } + for _, p := range ps { + if p != want { + return false + } + } + return true +} + +const samplePost = "From: alice@example.com\r\nSubject: hi\r\nMessage-ID: \r\n\r\nbody\r\n" + +func TestANewsletterIsDeliveredWithPriorityOne(t *testing.T) { + e := newsletterEnv(t) + res, err := e.r.prepare(e.list, []byte(samplePost)) + if err != nil { + t.Fatal(err) + } + if err := e.r.Deliver(res); err != nil { + t.Fatal(err) + } + if ps := duePriorities(t, e.s); !allEqual(ps, 1, 2) { + t.Fatalf("priorities: %v", ps) + } +} + +func TestADiscussionPostIsDeliveredWithPriorityZero(t *testing.T) { + e := newEnv(t) + if err := e.post(t, samplePost); err != nil { + t.Fatal(err) + } + if ps := duePriorities(t, e.s); !allEqual(ps, 0, 2) { + t.Fatalf("priorities: %v", ps) + } +} + +func TestAReleasedNewsletterHasPriorityOne(t *testing.T) { + e := newsletterEnv(t) + id, _ := e.s.HoldMessage(e.list.ID, "alice@example.com", []byte(samplePost), "r") + if err := e.r.Release(id); err != nil { + t.Fatal(err) + } + if ps := duePriorities(t, e.s); !allEqual(ps, 1, 2) { + t.Fatalf("priorities: %v", ps) + } +} + +func TestAReleasedDiscussionPostHasPriorityZero(t *testing.T) { + e := newEnv(t) + id, _ := e.s.HoldMessage(e.list.ID, "alice@example.com", []byte(samplePost), "r") + if err := e.r.Release(id); err != nil { + t.Fatal(err) + } + if ps := duePriorities(t, e.s); !allEqual(ps, 0, 2) { + t.Fatalf("priorities: %v", ps) + } +} + +func TestConfirmationMailOfANewsletterHasPriorityZero(t *testing.T) { + e := newsletterEnv(t) + if handled, err := e.r.Command("carol@x.com", "dev+join@example.org", nil); !handled || err != nil { + t.Fatalf("join: %v %v", handled, err) + } + if ps := duePriorities(t, e.s); !allEqual(ps, 0, 1) { + t.Fatalf("priorities: %v", ps) + } +} + +func TestAPostIsRecordedWithItsRecipients(t *testing.T) { + e := newEnv(t) + if err := e.post(t, samplePost); err != nil { + t.Fatal(err) + } + posts, err := e.s.ListPosts(e.list.ID, 0) + if err != nil || len(posts) != 1 || posts[0].Recipients != 2 || posts[0].MessageID != "" || posts[0].Subject == "" { + t.Fatalf("posts: %+v %v", posts, err) + } +} diff --git a/internal/router/router.go b/internal/router/router.go index 7e95813..cface5d 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -163,8 +163,13 @@ func postID(data []byte) string { return "sha256:" + hex.EncodeToString(sum[:]) } -// outbound builds one queue entry per recipient. +// outbound builds one queue entry per recipient. A newsletter (post policy publishers) is sent +// with priority 1, so that its many deliveries never delay discussion mail. func (r *Router) outbound(res Result) ([]store.NewOutbound, error) { + priority := 0 + if res.List.PostPolicy == store.PostPublishers { + priority = 1 + } items := make([]store.NewOutbound, 0, len(res.Recipients)) for _, rc := range res.Recipients { from := r.bounceAddress(res.List, rc.ID) @@ -172,7 +177,7 @@ func (r *Router) outbound(res Result) ([]store.NewOutbound, error) { return nil, errors.New("cannot create bounce address") } items = append(items, store.NewOutbound{ - From: from, Rcpt: rc.Address, MemberID: rc.ID, Prefix: r.unsubscribeHeaders(res.List, rc.ID), + From: from, Rcpt: rc.Address, MemberID: rc.ID, Prefix: r.unsubscribeHeaders(res.List, rc.ID), Priority: priority, }) } return items, nil From 2f2ee71f92217184a68c3960ca44e064b0c64919 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 12:40:55 +0100 Subject: [PATCH 4/7] feat(api): list the posts of a list with their delivery counters - GET /api/v1/lists/{id}/posts?limit= returns id, message_id, subject, created_at, recipients, sent, failed, dropped and pending, newest first; 404 for an unknown list, 400 for a bad limit, token required - discarding a queued delivery counts it as dropped (store.DropOutbox) - openapi: the new path and the discard note --- docs/openapi.yaml | 32 ++++++++++++ internal/api/api.go | 45 +++++++++++++++- internal/api/posts_test.go | 103 +++++++++++++++++++++++++++++++++++++ 3 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 internal/api/posts_test.go diff --git a/docs/openapi.yaml b/docs/openapi.yaml index ad0b348..a966df3 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -287,6 +287,7 @@ paths: - {name: id, in: path, required: true, schema: {type: integer}} delete: summary: Discard a queued delivery + description: A waiting delivery of a post counts as dropped in the post's counters; a failed one stays counted as failed. responses: '204': {description: Discarded} '404': {$ref: '#/components/responses/Error'} @@ -319,3 +320,34 @@ paths: errors: {type: array, items: {type: string}, description: The first 20 failures} '400': {$ref: '#/components/responses/Error'} '404': {$ref: '#/components/responses/Error'} + /lists/{id}/posts: + parameters: + - {name: id, in: path, required: true, schema: {type: integer}} + get: + summary: Posts queued to the list with their delivery counters, newest first + description: > + One entry per post queued to the list (delivered or approved), kept for 400 days. Confirmation mail is not a post. + Bodies are never read. + parameters: + - {name: limit, in: query, schema: {type: integer, default: 50, maximum: 200}, description: 0 means the default; more than 200 gives 200} + responses: + '200': + description: OK + content: + application/json: + schema: + type: array + items: + type: object + properties: + id: {type: integer} + message_id: {type: string, description: The Message-ID header, at most 200 characters; empty when there was none} + subject: {type: string, description: Decoded, at most 200 characters, control characters removed} + created_at: {type: string, format: date-time} + recipients: {type: integer, description: Deliveries queued (members at the time of the post)} + sent: {type: integer, description: Accepted by the relay} + failed: {type: integer, description: Given up for good (refused by the relay, or still failing after 3 days)} + dropped: {type: integer, description: The mailbox does not exist, or the delivery was discarded} + pending: {type: integer, description: "Still in the queue: recipients - sent - failed - dropped"} + '400': {$ref: '#/components/responses/Error'} + '404': {$ref: '#/components/responses/Error'} diff --git a/internal/api/api.go b/internal/api/api.go index eb03fd7..0a36925 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -75,6 +75,7 @@ func (s *server) routes() []route { {"GET /api/v1/queue", s.auth(s.getQueue)}, {"DELETE /api/v1/queue/{id}", s.auth(s.discardQueued)}, {"GET /api/v1/lists/{id}/held", s.auth(s.listHeld)}, + {"GET /api/v1/lists/{id}/posts", s.auth(s.listPosts)}, {"POST /api/v1/held/{id}/approve", s.auth(s.approveHeld)}, {"DELETE /api/v1/held/{id}", s.auth(s.discardHeld)}, } @@ -627,7 +628,7 @@ func (s *server) discardQueued(w http.ResponseWriter, r *http.Request) { if !ok { return } - if err := s.store.CompleteOutbox(id); err != nil { + if err := s.store.DropOutbox(id); err != nil { storeError(w, err) return } @@ -661,3 +662,45 @@ func (s *server) importArchive(w http.ResponseWriter, r *http.Request) { } writeJSON(w, http.StatusOK, sum) } + +// Handlers: posts. + +type postDTO struct { + ID int64 `json:"id"` + MessageID string `json:"message_id"` + Subject string `json:"subject"` + CreatedAt time.Time `json:"created_at"` + Recipients int `json:"recipients"` + Sent int `json:"sent"` + Failed int `json:"failed"` + Dropped int `json:"dropped"` + Pending int `json:"pending"` +} + +// listPosts returns the posts queued to a list with their delivery counters, newest first. +func (s *server) listPosts(w http.ResponseWriter, r *http.Request) { + id, ok := listID(w, r) + if !ok { + return + } + limit, ok := intParam(r, "limit", 0) // 0: the store's default + if !ok { + writeError(w, http.StatusBadRequest, "limit must be a non-negative integer") + return + } + if _, err := s.store.GetList(id); err != nil { + storeError(w, err) + return + } + ps, err := s.store.ListPosts(id, limit) + if err != nil { + storeError(w, err) + return + } + out := make([]postDTO, 0, len(ps)) + for _, p := range ps { + out = append(out, postDTO{ID: p.ID, MessageID: p.MessageID, Subject: p.Subject, CreatedAt: p.CreatedAt, + Recipients: p.Recipients, Sent: p.Sent, Failed: p.Failed, Dropped: p.Dropped, Pending: p.Pending}) + } + writeJSON(w, http.StatusOK, out) +} diff --git a/internal/api/posts_test.go b/internal/api/posts_test.go new file mode 100644 index 0000000..1e1b9dc --- /dev/null +++ b/internal/api/posts_test.go @@ -0,0 +1,103 @@ +package api + +import ( + "fmt" + "strings" + "testing" + "time" + + "codeberg.org/squidd/parsimony/internal/store" +) + +type postJSON struct { + ID int64 `json:"id"` + MessageID string `json:"message_id"` + Subject string `json:"subject"` + CreatedAt time.Time `json:"created_at"` + Recipients int `json:"recipients"` + Sent int `json:"sent"` + Failed int `json:"failed"` + Dropped int `json:"dropped"` + Pending int `json:"pending"` +} + +func TestListPosts(t *testing.T) { + h, s := queueServer(t) + now := time.Unix(1_700_000_000, 0) + s.Now = func() time.Time { return now } + l, _ := s.CreateList("dev", "example.org", "Dev") + rcpts := []store.NewOutbound{{From: "b@x.org", Rcpt: "a@x.com"}, {From: "b@x.org", Rcpt: "b@x.com"}, {From: "b@x.org", Rcpt: "c@x.com"}} + s.EnqueuePost(l.ID, "mid:<1@x>", []byte("Message-ID: <1@x>\r\nSubject: first\r\n\r\nsecret body"), rcpts) + now = now.Add(time.Minute) + s.EnqueuePost(l.ID, "mid:<2@x>", []byte("Message-ID: <2@x>\r\nSubject: second\r\n\r\nsecret body"), rcpts[:1]) + due, _ := s.DueOutbox(10) + s.CompleteOutbox(due[0].ID) + s.FailOutbox(due[1].ID, "no") + + rec := do(t, h, "GET", fmt.Sprintf("/api/v1/lists/%d/posts", l.ID), "", true) + expect(t, rec, 200) + var ps []postJSON + decode(t, rec, &ps) + if len(ps) != 2 || ps[0].Subject != "second" || ps[1].Subject != "first" { + t.Fatalf("newest first: %+v", ps) + } + p := ps[1] + if p.ID == 0 || p.MessageID != "<1@x>" || !p.CreatedAt.Equal(now.Add(-time.Minute)) || p.Recipients != 3 || + p.Sent != 1 || p.Failed != 1 || p.Dropped != 0 || p.Pending != 1 { + t.Fatalf("post: %+v", p) + } + if strings.Contains(rec.Body.String(), "secret body") { + t.Fatal("bodies are never listed") + } + + decode(t, do(t, h, "GET", fmt.Sprintf("/api/v1/lists/%d/posts?limit=1", l.ID), "", true), &ps) + if len(ps) != 1 || ps[0].Subject != "second" { + t.Fatalf("limit=1: %+v", ps) + } +} + +func TestListPostsEmptyIsAnArray(t *testing.T) { + h, s := queueServer(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + rec := do(t, h, "GET", fmt.Sprintf("/api/v1/lists/%d/posts", l.ID), "", true) + expect(t, rec, 200) + if strings.TrimSpace(rec.Body.String()) != "[]" { + t.Fatalf("body %q", rec.Body.String()) + } +} + +func TestListPostsLimit(t *testing.T) { + h, s := queueServer(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + for i := 0; i < 60; i++ { + s.EnqueuePost(l.ID, fmt.Sprintf("mid:%d", i), []byte("Subject: x\r\n\r\n"), nil) + } + path := fmt.Sprintf("/api/v1/lists/%d/posts", l.ID) + var ps []postJSON + decode(t, do(t, h, "GET", path, "", true), &ps) + if len(ps) != 50 { + t.Fatalf("default 50: %d", len(ps)) + } + for _, bad := range []string{"-1", "x", "1.5"} { + expect(t, do(t, h, "GET", path+"?limit="+bad, "", true), 400) + } +} + +func TestListPostsUnknownListAndAuth(t *testing.T) { + h, _ := queueServer(t) + expect(t, do(t, h, "GET", "/api/v1/lists/99/posts", "", true), 404) + expect(t, do(t, h, "GET", "/api/v1/lists/abc/posts", "", true), 404) + expect(t, do(t, h, "GET", "/api/v1/lists/99/posts", "", false), 401) +} + +func TestDiscardingADeliveryOfAPostCountsAsDropped(t *testing.T) { + h, s := queueServer(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.EnqueuePost(l.ID, "mid:<1@x>", []byte("Subject: x\r\n\r\n"), []store.NewOutbound{{From: "b@x.org", Rcpt: "a@x.com"}}) + due, _ := s.DueOutbox(10) + expect(t, do(t, h, "DELETE", fmt.Sprintf("/api/v1/queue/%d", due[0].ID), "", true), 204) + ps, _ := s.ListPosts(l.ID, 0) + if len(ps) != 1 || ps[0].Dropped != 1 || ps[0].Sent != 0 || ps[0].Pending != 0 { + t.Fatalf("posts: %+v", ps) + } +} From 7e43117c74a07616b633b466dd77124504783c65 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 12:42:15 +0100 Subject: [PATCH 5/7] feat(cli): parsimony posts ls - posts ls [--limit N] prints ID, DATE, SUBJECT, RECIPIENTS, SENT, PENDING, FAILED and DROPPED, newest first, "No posts yet." when empty, and the API's JSON with --json - cells go through the same control-character escaping as the other tables; tests for the table, the limit, JSON, errors and escaping --- internal/cli/cli.go | 3 ++ internal/cli/commands.go | 53 ++++++++++++++++++++++ internal/cli/posts_test.go | 91 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 147 insertions(+) create mode 100644 internal/cli/posts_test.go diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 2d50292..ff46225 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -49,6 +49,8 @@ Administration (talks to the REST API): archive import add the messages of an mbox file (Mailman export); safe to repeat, threads are rebuilt + posts ls [--limit N] posts sent to a list with their delivery counts + queue ls outbound queue queue rm discard one queued delivery @@ -105,6 +107,7 @@ func Run(args []string, io IO, getenv func(string) string) int { "held": {"ls": c.heldLs, "approve": c.heldApprove, "discard": c.heldDiscard}, "archive": {"ls": c.archiveLs, "show": c.archiveShow, "rm": c.archiveRm, "import": c.archiveImport}, "queue": {"ls": c.queueLs, "rm": c.queueRm}, + "posts": {"ls": c.postsLs}, "dkim": {"keygen": c.dkimKeygen, "record": c.dkimRecord, "check": c.dkimCheck}, } group, ok := commands[rest[0]] diff --git a/internal/cli/commands.go b/internal/cli/commands.go index 1232346..d9d7ee6 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -683,3 +683,56 @@ func (c *cmd) queueRm(args []string) int { return err }, "Discarded queued delivery %d") } + +// postsLs shows the posts queued to a list, newest first, with how far their delivery got. +func (c *cmd) postsLs(args []string) int { + fs := newFlagSet("posts ls") + limit := fs.Int("limit", 50, "") + pos, err := parseArgs(fs, args) + if err != nil { + return c.usageError("%v", err) + } + if len(pos) != 1 { + return c.usageError("usage: parsimony posts ls [--limit N]") + } + l, err := c.resolveList(pos[0]) + if err != nil { + return c.fail(err) + } + cl, _ := c.client() + raw, err := cl.do("GET", fmt.Sprintf("/lists/%d/posts?limit=%d", l.ID, *limit), nil) + if err != nil { + return c.fail(err) + } + if c.json { + return c.printJSON(raw) + } + var ps []struct { + ID int64 `json:"id"` + Subject string `json:"subject"` + CreatedAt time.Time `json:"created_at"` + Recipients int `json:"recipients"` + Sent int `json:"sent"` + Failed int `json:"failed"` + Dropped int `json:"dropped"` + Pending int `json:"pending"` + } + if err := json.Unmarshal(raw, &ps); err != nil { + return c.fail(err) + } + if len(ps) == 0 { + fmt.Fprintln(c.io.Out, "No posts yet.") + return 0 + } + rows := [][]string{{"ID", "DATE", "SUBJECT", "RECIPIENTS", "SENT", "PENDING", "FAILED", "DROPPED"}} + for _, p := range ps { + subject := p.Subject + if subject == "" { + subject = "(no subject)" + } + rows = append(rows, []string{strconv.FormatInt(p.ID, 10), formatDate(p.CreatedAt), subject, strconv.Itoa(p.Recipients), + strconv.Itoa(p.Sent), strconv.Itoa(p.Pending), strconv.Itoa(p.Failed), strconv.Itoa(p.Dropped)}) + } + c.table(rows) + return 0 +} diff --git a/internal/cli/posts_test.go b/internal/cli/posts_test.go new file mode 100644 index 0000000..ebc04b6 --- /dev/null +++ b/internal/cli/posts_test.go @@ -0,0 +1,91 @@ +package cli + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "codeberg.org/squidd/parsimony/internal/store" +) + +func TestPostsLs(t *testing.T) { + e := newEnv(t) + e.ok(t, "list", "add", "dev@example.org") + l, _ := e.s.FindList("dev@example.org") + if out := e.ok(t, "posts", "ls", "dev@example.org"); !strings.Contains(out, "No posts yet.") { + t.Fatalf("empty: %q", out) + } + now := time.Date(2026, 3, 1, 9, 30, 0, 0, time.UTC) + e.s.Now = func() time.Time { return now } + rcpts := []store.NewOutbound{{From: "b@x.org", Rcpt: "a@x.com"}, {From: "b@x.org", Rcpt: "b@x.com"}, {From: "b@x.org", Rcpt: "c@x.com"}, {From: "b@x.org", Rcpt: "d@x.com"}} + e.s.EnqueuePost(l.ID, "mid:<1@x>", []byte("Message-ID: <1@x>\r\nSubject: March issue\r\n\r\nb"), rcpts) + due, _ := e.s.DueOutbox(10) + e.s.CompleteOutbox(due[0].ID) + e.s.CompleteOutbox(due[1].ID) + e.s.FailOutbox(due[2].ID, "no") + now = now.Add(time.Hour) + e.s.EnqueuePost(l.ID, "mid:<2@x>", []byte("Message-ID: <2@x>\r\n\r\nb"), nil) + + out := e.ok(t, "posts", "ls", fmt.Sprint(l.ID)) + lines := strings.Split(strings.TrimSpace(out), "\n") + if len(lines) != 3 { + t.Fatalf("posts ls:\n%s", out) + } + if f := strings.Fields(lines[0]); strings.Join(f, " ") != "ID DATE SUBJECT RECIPIENTS SENT PENDING FAILED DROPPED" { + t.Fatalf("header: %q", lines[0]) + } + if f := strings.Fields(lines[1]); strings.Join(f, " ") != "2 2026-03-01 10:30 (no subject) 0 0 0 0 0" { + t.Fatalf("newest first: %q", lines[1]) + } + if f := strings.Fields(lines[2]); strings.Join(f, " ") != "1 2026-03-01 09:30 March issue 4 2 1 1 0" { + t.Fatalf("row: %q", lines[2]) + } + + if out := e.ok(t, "posts", "ls", "dev@example.org", "--limit", "1"); strings.Contains(out, "March issue") || !strings.Contains(out, "(no subject)") { + t.Fatalf("--limit 1:\n%s", out) + } + var js []map[string]any + if err := json.Unmarshal([]byte(e.ok(t, "--json", "posts", "ls", "dev@example.org")), &js); err != nil || len(js) != 2 || + js[1]["subject"] != "March issue" || js[1]["pending"] != float64(1) || js[1]["message_id"] != "<1@x>" { + t.Fatalf("json: %v %v", js, err) + } +} + +func TestPostsLsErrors(t *testing.T) { + e := newEnv(t) + e.fails(t, 2, "usage: parsimony posts ls", "posts", "ls") + e.fails(t, 2, "usage: parsimony posts ls", "posts", "ls", "a@x", "b@x") + e.fails(t, 2, "flag provided but not defined", "posts", "ls", "a@x", "--nope") + e.fails(t, 1, "no such list", "posts", "ls", "nope@example.org") + e.ok(t, "list", "add", "dev@example.org") + e.fails(t, 1, "limit must be", "posts", "ls", "dev@example.org", "--limit", "-1") + if out := e.ok(t, "help"); !strings.Contains(out, "posts ls [--limit N]") { + t.Fatalf("help lacks posts ls:\n%s", out) + } +} + +func TestPostsLsEscapesControlCharacters(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/api/v1/lists/1": + fmt.Fprint(w, `{"id":1,"address":"dev@example.org"}`) + case "/api/v1/lists/1/posts": + b, _ := json.Marshal([]map[string]any{{"id": 1, "subject": "s " + esc, "created_at": time.Unix(0, 0), "recipients": 1}}) + w.Write(b) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + e := &env{srv: srv} + out := e.ok(t, "posts", "ls", "1") + noControl(t, "posts ls", out) + if !strings.Contains(out, `\x1b[31mRED`) { + t.Fatalf("the escape sequence should stay visible as text:\n%s", out) + } +} From ee1f837a207761f35686705dfe7d28cfb0fe7760 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 12:43:55 +0100 Subject: [PATCH 6/7] feat(admin): a Sent tab with the posts of a list and their progress - route /lists/:id/sent and a Sent tab on the list page - a table of posts (subject, date, sent of recipients with a progress bar, recipients, pending, failed, dropped) and an empty state; a post with no recipients shows no bar - refreshes every 5 seconds while any post has pending deliveries, and stops when none has - api.posts.list, the Post type and the fake API route; tests --- admin/src/App.tsx | 2 + admin/src/lib/api.ts | 19 ++++++ admin/src/pages/ListPage.tsx | 3 +- admin/src/tabs/SentTab.test.tsx | 113 ++++++++++++++++++++++++++++++++ admin/src/tabs/SentTab.tsx | 81 +++++++++++++++++++++++ admin/src/test/fakeApi.ts | 10 ++- 6 files changed, 225 insertions(+), 3 deletions(-) create mode 100644 admin/src/tabs/SentTab.test.tsx create mode 100644 admin/src/tabs/SentTab.tsx diff --git a/admin/src/App.tsx b/admin/src/App.tsx index e832cba..91339ba 100644 --- a/admin/src/App.tsx +++ b/admin/src/App.tsx @@ -7,6 +7,7 @@ import { NotFoundPage } from '@/pages/NotFoundPage' import { ArchiveTab } from '@/tabs/ArchiveTab' import { HeldTab } from '@/tabs/HeldTab' import { MembersTab } from '@/tabs/MembersTab' +import { SentTab } from '@/tabs/SentTab' import { SettingsTab } from '@/tabs/SettingsTab' import { QueuePage } from '@/pages/QueuePage' import { Toaster } from '@/components/ui/sonner' @@ -25,6 +26,7 @@ export function App() { } /> } /> } /> + } /> } /> diff --git a/admin/src/lib/api.ts b/admin/src/lib/api.ts index 0f25352..5f1d4a2 100644 --- a/admin/src/lib/api.ts +++ b/admin/src/lib/api.ts @@ -79,6 +79,22 @@ export interface Queue { items: QueueItem[] } +/** A post queued to a list, with how far its delivery got. */ +export interface Post { + id: number + message_id: string + subject: string + created_at: string + /** Deliveries queued: the members when it was posted. */ + recipients: number + sent: number + failed: number + /** The mailbox does not exist, or the delivery was discarded. */ + dropped: number + /** Still in the queue. */ + pending: number +} + export class ApiError extends Error { readonly status: number constructor(status: number, message: string) { @@ -164,6 +180,9 @@ export const api = { approve: (id: number) => request('POST', `/held/${id}/approve`), discard: (id: number) => request('DELETE', `/held/${id}`), }, + posts: { + list: (listId: number) => request('GET', `/lists/${listId}/posts`), + }, queue: { get: () => request('GET', '/queue'), discard: (id: number) => request('DELETE', `/queue/${id}`), diff --git a/admin/src/pages/ListPage.tsx b/admin/src/pages/ListPage.tsx index 4d65251..77ff633 100644 --- a/admin/src/pages/ListPage.tsx +++ b/admin/src/pages/ListPage.tsx @@ -4,7 +4,7 @@ import { Badge } from '@/components/ui/badge' import { Tabs, TabsList, TabsTrigger } from '@/components/ui/tabs' import { api } from '@/lib/api' -const tabs = ['settings', 'members', 'held', 'archive'] as const +const tabs = ['settings', 'members', 'held', 'archive', 'sent'] as const export function ListPage() { const params = useParams() @@ -42,6 +42,7 @@ export function ListPage() { {heldCount > 0 && {heldCount}} Archive + Sent diff --git a/admin/src/tabs/SentTab.test.tsx b/admin/src/tabs/SentTab.test.tsx new file mode 100644 index 0000000..35fc02f --- /dev/null +++ b/admin/src/tabs/SentTab.test.tsx @@ -0,0 +1,113 @@ +import { act, screen, within } from '@testing-library/react' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { Post } from '@/lib/api' +import { aList, installFakeApi, newState } from '@/test/fakeApi' +import { renderApp } from '@/test/render' + +const post = (id: number, over: Partial = {}): Post => ({ + id, + message_id: `<${id}@example.org>`, + subject: `Issue ${id}`, + created_at: '2030-01-02T03:04:05Z', + recipients: 4, + sent: 4, + failed: 0, + dropped: 0, + pending: 0, + ...over, +}) + +async function setup(posts: Post[]) { + const state = installFakeApi(newState({ lists: [aList({ id: 1 })], posts: { 1: posts } })) + const ctx = renderApp('/lists/1/sent', { token: 'good' }) + await screen.findByRole('heading', { name: /sent posts/i, level: 2 }) + return { ...ctx, state } +} + +const postCalls = (calls: string[]) => calls.filter((c) => c === 'GET /lists/1/posts').length + +afterEach(() => vi.useRealTimers()) + +describe('sent tab', () => { + it('is a tab of the list page', async () => { + installFakeApi(newState({ lists: [aList({ id: 1 })], posts: { 1: [post(1)] } })) + const { user } = renderApp('/lists/1', { token: 'good' }) + await user.click(await screen.findByRole('tab', { name: /^sent$/i })) + expect(await screen.findByRole('heading', { name: /sent posts/i, level: 2 })).toBeInTheDocument() + expect(await screen.findByText('Issue 1')).toBeInTheDocument() + expect(screen.getByRole('tab', { name: /^sent$/i })).toHaveAttribute('aria-selected', 'true') + }) + + it('says when nothing was sent yet', async () => { + await setup([]) + expect(await screen.findByText(/nothing has been sent to this list yet/i)).toBeInTheDocument() + expect(screen.queryByRole('table')).not.toBeInTheDocument() + }) + + it('shows each post with its counts and progress', async () => { + await setup([ + post(2, { subject: '', recipients: 10, sent: 6, failed: 1, dropped: 1, pending: 2 }), + post(1), + ]) + const row = (await screen.findByText('(no subject)')).closest('tr')! + expect(within(row).getByText('2030-01-02 03:04 UTC')).toBeInTheDocument() + const bar = within(row).getByRole('progressbar') + expect(bar).toHaveAttribute('aria-valuenow', '6') + expect(bar).toHaveAttribute('aria-valuemax', '10') + expect(within(row).getByText('6 of 10')).toBeInTheDocument() + const cells = within(row).getAllByRole('cell').map((c) => c.textContent) + expect(cells.slice(3)).toEqual(['10', '2', '1', '1']) + const headers = screen.getAllByRole('columnheader').map((h) => h.textContent) + expect(headers).toEqual(['Subject', 'Date', 'Sent', 'Recipients', 'Pending', 'Failed', 'Dropped']) + const done = screen.getByText('Issue 1').closest('tr')! + expect(within(done).getByRole('progressbar')).toHaveAttribute('aria-valuenow', '4') + const rows = screen.getAllByRole('row').slice(1).map((r) => r.textContent ?? '') + expect(rows[0]).toContain('(no subject)') // the order of the server: newest first + }) + + it('shows a post nobody received without a progress bar', async () => { + await setup([post(1, { recipients: 0, sent: 0 })]) + const row = (await screen.findByText('Issue 1')).closest('tr')! + expect(within(row).queryByRole('progressbar')).not.toBeInTheDocument() + expect(within(row).getByText(/no recipients/i)).toBeInTheDocument() + }) + + it('shows subjects as text', async () => { + await setup([post(1, { subject: '' })]) + expect(await screen.findByText('')).toBeInTheDocument() + expect(document.querySelector('img[src="x"]')).toBeNull() + }) + + it('refreshes every few seconds while a post is being sent', async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }) + const { state } = await setup([post(1, { sent: 1, pending: 3 })]) + await screen.findByText('1 of 4') + state.posts[1] = [post(1)] + await act(async () => { + await vi.advanceTimersByTimeAsync(6000) + }) + expect(await screen.findByText('4 of 4')).toBeInTheDocument() + // Everything is sent: no more polling. + const before = postCalls(state.calls) + await act(async () => { + await vi.advanceTimersByTimeAsync(12000) + }) + expect(postCalls(state.calls)).toBe(before) + }) + + it('does not refresh when nothing is pending', async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }) + const { state } = await setup([post(1)]) + await screen.findByText('4 of 4') + await act(async () => { + await vi.advanceTimersByTimeAsync(12000) + }) + expect(postCalls(state.calls)).toBe(1) + }) + + it('shows a loading error', async () => { + installFakeApi(newState({ lists: [aList({ id: 1 })], fail: { 'GET /lists/1/posts': { status: 500, error: 'database is locked' } } })) + renderApp('/lists/1/sent', { token: 'good' }) + expect(await screen.findByRole('alert')).toHaveTextContent('database is locked') + }) +}) diff --git a/admin/src/tabs/SentTab.tsx b/admin/src/tabs/SentTab.tsx new file mode 100644 index 0000000..9426b0e --- /dev/null +++ b/admin/src/tabs/SentTab.tsx @@ -0,0 +1,81 @@ +import { useQuery } from '@tanstack/react-query' +import { useOutletContext } from 'react-router' +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table' +import { api, type List, type Post } from '@/lib/api' +import { formatDate } from '@/lib/format' + +/** How often the tab reloads by itself while a post is still being sent. */ +export const REFRESH_MS = 5000 + +export function SentTab() { + const list = useOutletContext() + const posts = useQuery({ + queryKey: ['posts', list.id], + queryFn: () => api.posts.list(list.id), + refetchInterval: (q) => (q.state.data?.some((p) => p.pending > 0) ? REFRESH_MS : false), + }) + const items = posts.data ?? [] + + return ( +
+

Sent posts

+ {posts.isPending &&

Loading…

} + {posts.isError &&

{posts.error.message}

} + {posts.data && items.length === 0 && ( +

Nothing has been sent to this list yet.

+ )} + {items.length > 0 && ( + + + + Subject + Date + Sent + Recipients + Pending + Failed + Dropped + + + + {items.map((p) => ( + + {p.subject || '(no subject)'} + {formatDate(p.created_at)} + + + + {p.recipients} + {p.pending} + {p.failed} + {p.dropped} + + ))} + +
+ )} +
+ ) +} + +function Progress({ post }: { post: Post }) { + if (post.recipients === 0) return No recipients + const pct = Math.round((100 * post.sent) / post.recipients) + return ( +
+
+
+
+ + {post.sent} of {post.recipients} + +
+ ) +} diff --git a/admin/src/test/fakeApi.ts b/admin/src/test/fakeApi.ts index c8bbfaf..b18176b 100644 --- a/admin/src/test/fakeApi.ts +++ b/admin/src/test/fakeApi.ts @@ -1,6 +1,6 @@ // An in-memory stand-in for the Parsimony API, used by component tests. import { vi } from 'vitest' -import type { Held, List, Member, Message, QueueItem } from '@/lib/api' +import type { Held, List, Member, Message, Post, QueueItem } from '@/lib/api' export interface FakeState { token: string @@ -9,6 +9,7 @@ export interface FakeState { held: Record archive: Record queue: QueueItem[] + posts: Record /** Like the server, the queue listing is capped. */ queueLimit: number /** Content types seen on archive imports. */ @@ -23,7 +24,7 @@ export interface FakeState { } export function newState(overrides: Partial = {}): FakeState { - return { token: 'good', lists: [], members: {}, held: {}, archive: {}, queue: [], queueLimit: 100, importTypes: [], nextId: 100, calls: [], fail: {}, ...overrides } + return { token: 'good', lists: [], members: {}, held: {}, archive: {}, queue: [], posts: {}, queueLimit: 100, importTypes: [], nextId: 100, calls: [], fail: {}, ...overrides } } export function aList(over: Partial = {}): List { @@ -125,6 +126,11 @@ export function installFakeApi(state: FakeState) { state.members[id] = before.filter((x) => x.address !== addr) return json(204) } + if ((m = path.match(/^\/lists\/(\d+)\/posts$/)) && method === 'GET') { + const id = Number(m[1]) + if (!state.lists.some((l) => l.id === id)) return err(404, 'not found') + return json(200, state.posts[id] ?? []) + } if ((m = path.match(/^\/lists\/(\d+)\/held$/)) && method === 'GET') return json(200, state.held[Number(m[1])] ?? []) if ((m = path.match(/^\/held\/(\d+)(\/approve)?$/))) { const id = Number(m[1]) From 47dc118b9858dd0aaf0170566d9f3ba8d658e1e7 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 12:47:17 +0100 Subject: [PATCH 7/7] test: tighten the posts tests found weak by mutation - ListPosts: a limit of 201 is capped to 200 - posts ls: distinct pending, failed and dropped counts, so a swapped column is caught --- internal/cli/posts_test.go | 16 ++++++++++------ internal/store/posts_test.go | 3 +++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/internal/cli/posts_test.go b/internal/cli/posts_test.go index ebc04b6..ed8b597 100644 --- a/internal/cli/posts_test.go +++ b/internal/cli/posts_test.go @@ -21,12 +21,16 @@ func TestPostsLs(t *testing.T) { } now := time.Date(2026, 3, 1, 9, 30, 0, 0, time.UTC) e.s.Now = func() time.Time { return now } - rcpts := []store.NewOutbound{{From: "b@x.org", Rcpt: "a@x.com"}, {From: "b@x.org", Rcpt: "b@x.com"}, {From: "b@x.org", Rcpt: "c@x.com"}, {From: "b@x.org", Rcpt: "d@x.com"}} + var rcpts []store.NewOutbound + for i := 0; i < 7; i++ { + rcpts = append(rcpts, store.NewOutbound{From: "b@x.org", Rcpt: fmt.Sprintf("u%d@x.com", i)}) + } e.s.EnqueuePost(l.ID, "mid:<1@x>", []byte("Message-ID: <1@x>\r\nSubject: March issue\r\n\r\nb"), rcpts) due, _ := e.s.DueOutbox(10) - e.s.CompleteOutbox(due[0].ID) - e.s.CompleteOutbox(due[1].ID) - e.s.FailOutbox(due[2].ID, "no") + for _, d := range due[:4] { + e.s.CompleteOutbox(d.ID) + } + e.s.FailOutbox(due[4].ID, "no") now = now.Add(time.Hour) e.s.EnqueuePost(l.ID, "mid:<2@x>", []byte("Message-ID: <2@x>\r\n\r\nb"), nil) @@ -41,7 +45,7 @@ func TestPostsLs(t *testing.T) { if f := strings.Fields(lines[1]); strings.Join(f, " ") != "2 2026-03-01 10:30 (no subject) 0 0 0 0 0" { t.Fatalf("newest first: %q", lines[1]) } - if f := strings.Fields(lines[2]); strings.Join(f, " ") != "1 2026-03-01 09:30 March issue 4 2 1 1 0" { + if f := strings.Fields(lines[2]); strings.Join(f, " ") != "1 2026-03-01 09:30 March issue 7 4 2 1 0" { t.Fatalf("row: %q", lines[2]) } @@ -50,7 +54,7 @@ func TestPostsLs(t *testing.T) { } var js []map[string]any if err := json.Unmarshal([]byte(e.ok(t, "--json", "posts", "ls", "dev@example.org")), &js); err != nil || len(js) != 2 || - js[1]["subject"] != "March issue" || js[1]["pending"] != float64(1) || js[1]["message_id"] != "<1@x>" { + js[1]["subject"] != "March issue" || js[1]["pending"] != float64(2) || js[1]["message_id"] != "<1@x>" { t.Fatalf("json: %v %v", js, err) } } diff --git a/internal/store/posts_test.go b/internal/store/posts_test.go index 1c9bce1..24a4960 100644 --- a/internal/store/posts_test.go +++ b/internal/store/posts_test.go @@ -268,6 +268,9 @@ func TestListPostsNewestFirstWithALimit(t *testing.T) { if some, _ := s.ListPosts(dev.ID, 7); len(some) != 7 { t.Fatalf("limit 7: %d", len(some)) } + if over, _ := s.ListPosts(dev.ID, 201); len(over) != 200 { + t.Fatalf("limit 201: %d", len(over)) + } if exact, _ := s.ListPosts(dev.ID, 200); len(exact) != 200 { t.Fatalf("limit 200: %d", len(exact)) }