diff --git a/internal/router/identity_test.go b/internal/router/identity_test.go index 0084f1c..47c5c02 100644 --- a/internal/router/identity_test.go +++ b/internal/router/identity_test.go @@ -4,6 +4,8 @@ import ( "errors" "strings" "testing" + + "codeberg.org/squidd/parsimony/internal/store" ) func postFrom(from string) []byte { @@ -109,3 +111,30 @@ func TestAFullModerationQueueRefusesTemporarily(t *testing.T) { t.Fatalf("want ErrHeldFull, got %v", err) } } + +func TestSimultaneousApprovalsDeliverOnce(t *testing.T) { + e := newEnv(t) + e.s.SetPolicies(e.list.ID, "moderated", "open", "private") + e.r.Route("alice@example.com", "dev@example.org", postFrom("alice@example.com")) + held, _ := e.s.ListHeld(e.list.ID) + + const n = 8 + results := make(chan error, n) + for i := 0; i < n; i++ { + go func() { results <- e.r.Release(held[0].ID) }() + } + wins := 0 + for i := 0; i < n; i++ { + if err := <-results; err == nil { + wins++ + } else if !errors.Is(err, store.ErrNotFound) { + t.Fatalf("unexpected error: %v", err) + } + } + if wins != 1 { + t.Fatalf("%d approvals succeeded, want exactly one", wins) + } + if st, _ := e.s.OutboxStats(); st.Pending != 2 { + t.Fatalf("2 members, one approval: pending = %d", st.Pending) + } +} diff --git a/internal/router/router.go b/internal/router/router.go index 697e5fa..1be4115 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -126,17 +126,26 @@ func (r *Router) prepare(l store.List, data []byte) (Result, error) { // bounce address (VERP) and unsubscribe headers. The queue write is atomic: on // error nothing was queued and the caller may safely retry. func (r *Router) Deliver(res Result) error { + items, err := r.outbound(res) + if err != nil { + return err + } + return r.enqueue(res.Data, items) +} + +// outbound builds one queue entry per recipient. +func (r *Router) outbound(res Result) ([]store.NewOutbound, error) { items := make([]store.NewOutbound, 0, len(res.Recipients)) for _, rc := range res.Recipients { from := r.bounceAddress(res.List, rc.ID) if from == "" { - return errors.New("cannot create bounce address") + return nil, errors.New("cannot create bounce address") } items = append(items, store.NewOutbound{ From: from, Rcpt: rc.Address, MemberID: rc.ID, Prefix: r.unsubscribeHeaders(res.List, rc.ID), }) } - return r.enqueue(res.Data, items) + return items, nil } // enqueue stores deliveries and wakes the sender. @@ -183,8 +192,8 @@ func (r *Router) archive(l store.List, data []byte) { } } -// Release approves a held message: it is queued for delivery and leaves the -// moderation queue. +// Release approves a held message: it leaves the moderation queue and is queued for delivery in +// one transaction, so simultaneous approvals deliver it once (the others get store.ErrNotFound). func (r *Router) Release(heldID int64) error { h, err := r.store.GetHeld(heldID) if err != nil { @@ -198,10 +207,17 @@ func (r *Router) Release(heldID int64) error { if err != nil { return err } - if err := r.Deliver(res); err != nil { + items, err := r.outbound(res) + if err != nil { return err } - return r.store.DeleteHeld(heldID) + if err := r.store.EnqueueFromHeld(heldID, res.Data, items); err != nil { + return err + } + if r.OnEnqueue != nil && len(items) > 0 { + r.OnEnqueue() + } + return nil } // fromHeaderAddress returns the lowercase address of the From header when there is exactly one. diff --git a/internal/store/archive_test.go b/internal/store/archive_test.go index c8fb809..e44b17f 100644 --- a/internal/store/archive_test.go +++ b/internal/store/archive_test.go @@ -2,7 +2,6 @@ package store import ( "errors" - "fmt" "strings" "testing" "time" @@ -410,25 +409,39 @@ func TestRethreadCycleRootIsTheSmallestIdEvenWhenEnteredFromElsewhere(t *testing } } -func TestRethreadIsLinearOnLongChains(t *testing.T) { - s := newTestStore(t) - l, _ := s.CreateList("dev", "example.org", "Dev") - const n = 20000 - ms := make([]Message, n) - for i := range ms { - ms[i] = msg(fmt.Sprintf("", i), "S", "b") +func TestThreadRootsIsLinearOnLongChains(t *testing.T) { + const n = 300_000 + ids := make([]int64, n) + parent := make(map[int64]int64, n) + for i := range ids { + ids[i] = int64(i + 1) if i > 0 { - ms[i].InReplyTo = fmt.Sprintf("", i-1) + parent[int64(i+1)] = int64(i) } } - if _, err := s.ArchiveBatch(l.ID, ms); err != nil { - t.Fatal(err) - } - done := make(chan struct{}) - go func() { s.Rethread(l.ID); close(done) }() + done := make(chan map[int64]int64, 1) + go func() { done <- threadRoots(ids, parent) }() select { - case <-done: + case roots := <-done: + if roots[n] != 1 || roots[1] != 1 || roots[n/2] != 1 { + t.Fatalf("everything is in the thread of message 1: %d %d %d", roots[n], roots[1], roots[n/2]) + } case <-time.After(5 * time.Second): - t.Fatal("Rethread is not linear: a 20000-message chain took more than 5 seconds") + t.Fatal("threadRoots is not linear: a 300000-message chain took more than 5 seconds") + } +} + +func TestThreadRootsFromTheNewestEndOfAChain(t *testing.T) { + // Visiting the newest message first must give the same roots as visiting the oldest first. + parent := map[int64]int64{2: 1, 3: 2, 4: 3, 6: 5} + forward := threadRoots([]int64{1, 2, 3, 4, 5, 6}, parent) + backward := threadRoots([]int64{6, 5, 4, 3, 2, 1}, parent) + for _, id := range []int64{1, 2, 3, 4, 5, 6} { + if forward[id] != backward[id] { + t.Fatalf("order dependent: %d -> %d vs %d", id, forward[id], backward[id]) + } + } + if forward[4] != 1 || forward[6] != 5 || forward[5] != 5 { + t.Fatalf("roots: %v", forward) } } diff --git a/internal/store/outbox.go b/internal/store/outbox.go index 37790c5..34f0419 100644 --- a/internal/store/outbox.go +++ b/internal/store/outbox.go @@ -45,12 +45,39 @@ func (s *Store) Enqueue(msg []byte, items []NewOutbound) error { return err } defer tx.Rollback() + if err := enqueueTx(tx, s.Now().Unix(), msg, items); err != nil { + return err + } + return tx.Commit() +} + +// EnqueueFromHeld releases a held message: it removes it from the moderation queue and queues +// its deliveries in one transaction. Of several simultaneous approvals only one succeeds, the +// others get ErrNotFound, so a message is never delivered twice. With no recipients the message +// is still released. +func (s *Store) EnqueueFromHeld(heldID int64, msg []byte, items []NewOutbound) error { + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + if err := affected(tx.Exec(`DELETE FROM held WHERE id=?`, heldID)); err != nil { + return err + } + if len(items) > 0 { + if err := enqueueTx(tx, s.Now().Unix(), msg, items); err != nil { + return err + } + } + return tx.Commit() +} + +func enqueueTx(tx *sql.Tx, now int64, msg []byte, items []NewOutbound) error { res, err := tx.Exec(`INSERT INTO outbox_msgs(data) VALUES(?)`, msg) if err != nil { return err } msgID, _ := res.LastInsertId() - now := s.Now().Unix() for _, it := range items { prefix := it.Prefix if prefix == nil { @@ -61,7 +88,7 @@ func (s *Store) Enqueue(msg []byte, items []NewOutbound) error { return err } } - return tx.Commit() + return nil } const outboxCols = `id, msg_id, envelope_from, rcpt, member_id, prefix, attempts, next_at, last_error, created_at` diff --git a/internal/store/outbox_test.go b/internal/store/outbox_test.go index 4ec14bd..d30e8cf 100644 --- a/internal/store/outbox_test.go +++ b/internal/store/outbox_test.go @@ -188,3 +188,74 @@ func TestQueueSurvivesReopen(t *testing.T) { t.Fatalf("data: %q", data) } } + +func TestEnqueueFromHeldMovesTheMessageAtomically(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("Subject: s\r\n\r\nb"), "r") + + if err := s.EnqueueFromHeld(hid, []byte("final"), items(2)); err != nil { + t.Fatal(err) + } + if _, err := s.GetHeld(hid); !errors.Is(err, ErrNotFound) { + t.Fatal("the held message is gone") + } + if st, _ := s.OutboxStats(); st.Pending != 2 { + t.Fatalf("pending = %d", st.Pending) + } + // Approving again must not queue anything. + if err := s.EnqueueFromHeld(hid, []byte("final"), items(2)); !errors.Is(err, ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } + if st, _ := s.OutboxStats(); st.Pending != 2 { + t.Fatalf("a second approval queued more mail: %d", st.Pending) + } +} + +func TestEnqueueFromHeldWithNoRecipientsStillReleasesTheMessage(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r") + if err := s.EnqueueFromHeld(hid, []byte("final"), nil); err != nil { + t.Fatal(err) + } + if _, err := s.GetHeld(hid); !errors.Is(err, ErrNotFound) { + t.Fatal("the message is released even if nobody is subscribed") + } + if st, _ := s.OutboxStats(); st.Pending != 0 { + t.Fatalf("pending = %d", st.Pending) + } +} + +func TestOnlyOneOfSeveralSimultaneousApprovalsWins(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r") + const n = 12 + results := make(chan error, n) + start := make(chan struct{}) + for i := 0; i < n; i++ { + go func() { + <-start + results <- s.EnqueueFromHeld(hid, []byte("final"), items(3)) + }() + } + close(start) + wins, notFound := 0, 0 + for i := 0; i < n; i++ { + switch err := <-results; { + case err == nil: + wins++ + case errors.Is(err, ErrNotFound): + notFound++ + default: + t.Fatalf("unexpected error: %v", err) + } + } + if wins != 1 || notFound != n-1 { + t.Fatalf("wins=%d notFound=%d", wins, notFound) + } + if st, _ := s.OutboxStats(); st.Pending != 3 { + t.Fatalf("the message must be queued exactly once for 3 recipients, pending = %d", st.Pending) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 0c0015f..e1e205b 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -709,6 +709,47 @@ func archiveOne(q querier, listID int64, m Message) (id int64, created bool, err return id, true, nil } +// threadRoots maps every id to the root of its reply chain. parent[id] is the message id replies to, +// when that message exists. A cycle of replies has the smallest id of the cycle as root. Each chain +// is walked once, so the cost is linear in the number of messages. +func threadRoots(ids []int64, parent map[int64]int64) map[int64]int64 { + roots := make(map[int64]int64, len(ids)) + for _, start := range ids { + if _, done := roots[start]; done { + continue + } + path := []int64{start} + seen := map[int64]int{start: 0} + cur := start + var result int64 + for { + p, ok := parent[cur] + if !ok { + result = cur + break + } + if r, known := roots[p]; known { + result = r + break + } + if at, loop := seen[p]; loop { // a cycle: its smallest id is the root + result = path[at] + for _, id := range path[at:] { + result = min(result, id) + } + break + } + seen[p] = len(path) + path = append(path, p) + cur = p + } + for _, id := range path { + roots[id] = result + } + } + return roots +} + // Rethread recomputes the thread of every message of a list from the In-Reply-To // links, and returns how many messages changed thread. Insertion order does not // matter: a message imported before its parent ends up in the parent's thread. @@ -740,46 +781,14 @@ func (s *Store) Rethread(listID int64) (int, error) { } parent := map[int64]int64{} + ids := make([]int64, 0, len(all)) for _, r := range all { + ids = append(ids, r.id) if p, ok := byMsgID[r.replyTo]; ok && r.replyTo != "" { parent[r.id] = p } } - roots := map[int64]int64{} - root := func(start int64) int64 { - if r, ok := roots[start]; ok { - return r - } - path := []int64{start} - seen := map[int64]int{start: 0} - cur := start - var result int64 - for { - p, ok := parent[cur] - if !ok { - result = cur - break - } - if r, known := roots[p]; known { - result = r - break - } - if at, loop := seen[p]; loop { // a cycle: its smallest id is the root - result = path[at] - for _, id := range path[at:] { - result = min(result, id) - } - break - } - seen[p] = len(path) - path = append(path, p) - cur = p - } - for _, id := range path { - roots[id] = result - } - return result - } + roots := threadRoots(ids, parent) tx, err := s.db.Begin() if err != nil { @@ -788,7 +797,7 @@ func (s *Store) Rethread(listID int64) (int, error) { defer tx.Rollback() changed := 0 for _, r := range all { - if want := root(r.id); want != r.thread { + if want := roots[r.id]; want != r.thread { if _, err := tx.Exec(`UPDATE archive SET thread_id=? WHERE id=?`, want, r.id); err != nil { return 0, err }