fix: approve a held message once, however many approvals race

- Release removes the held message and queues its deliveries in one transaction
- Of several simultaneous approvals one wins, the others get 404
- Extract the thread-root computation and test its linearity without the database (fast under -race)
This commit is contained in:
tgiachi 2026-11-09 20:29:42 +01:00
commit e7e1afbde9
6 changed files with 225 additions and 60 deletions

View file

@ -4,6 +4,8 @@ import (
"errors"
"strings"
"testing"
"codeberg.org/squidd/parsimony/internal/store"
)
func postFrom(from string) []byte {
@ -109,3 +111,30 @@ func TestAFullModerationQueueRefusesTemporarily(t *testing.T) {
t.Fatalf("want ErrHeldFull, got %v", err)
}
}
func TestSimultaneousApprovalsDeliverOnce(t *testing.T) {
e := newEnv(t)
e.s.SetPolicies(e.list.ID, "moderated", "open", "private")
e.r.Route("alice@example.com", "dev@example.org", postFrom("alice@example.com"))
held, _ := e.s.ListHeld(e.list.ID)
const n = 8
results := make(chan error, n)
for i := 0; i < n; i++ {
go func() { results <- e.r.Release(held[0].ID) }()
}
wins := 0
for i := 0; i < n; i++ {
if err := <-results; err == nil {
wins++
} else if !errors.Is(err, store.ErrNotFound) {
t.Fatalf("unexpected error: %v", err)
}
}
if wins != 1 {
t.Fatalf("%d approvals succeeded, want exactly one", wins)
}
if st, _ := e.s.OutboxStats(); st.Pending != 2 {
t.Fatalf("2 members, one approval: pending = %d", st.Pending)
}
}

View file

@ -126,17 +126,26 @@ func (r *Router) prepare(l store.List, data []byte) (Result, error) {
// bounce address (VERP) and unsubscribe headers. The queue write is atomic: on
// error nothing was queued and the caller may safely retry.
func (r *Router) Deliver(res Result) error {
items, err := r.outbound(res)
if err != nil {
return err
}
return r.enqueue(res.Data, items)
}
// outbound builds one queue entry per recipient.
func (r *Router) outbound(res Result) ([]store.NewOutbound, error) {
items := make([]store.NewOutbound, 0, len(res.Recipients))
for _, rc := range res.Recipients {
from := r.bounceAddress(res.List, rc.ID)
if from == "" {
return errors.New("cannot create bounce address")
return nil, errors.New("cannot create bounce address")
}
items = append(items, store.NewOutbound{
From: from, Rcpt: rc.Address, MemberID: rc.ID, Prefix: r.unsubscribeHeaders(res.List, rc.ID),
})
}
return r.enqueue(res.Data, items)
return items, nil
}
// enqueue stores deliveries and wakes the sender.
@ -183,8 +192,8 @@ func (r *Router) archive(l store.List, data []byte) {
}
}
// Release approves a held message: it is queued for delivery and leaves the
// moderation queue.
// Release approves a held message: it leaves the moderation queue and is queued for delivery in
// one transaction, so simultaneous approvals deliver it once (the others get store.ErrNotFound).
func (r *Router) Release(heldID int64) error {
h, err := r.store.GetHeld(heldID)
if err != nil {
@ -198,10 +207,17 @@ func (r *Router) Release(heldID int64) error {
if err != nil {
return err
}
if err := r.Deliver(res); err != nil {
items, err := r.outbound(res)
if err != nil {
return err
}
return r.store.DeleteHeld(heldID)
if err := r.store.EnqueueFromHeld(heldID, res.Data, items); err != nil {
return err
}
if r.OnEnqueue != nil && len(items) > 0 {
r.OnEnqueue()
}
return nil
}
// fromHeaderAddress returns the lowercase address of the From header when there is exactly one.

View file

@ -2,7 +2,6 @@ package store
import (
"errors"
"fmt"
"strings"
"testing"
"time"
@ -410,25 +409,39 @@ func TestRethreadCycleRootIsTheSmallestIdEvenWhenEnteredFromElsewhere(t *testing
}
}
func TestRethreadIsLinearOnLongChains(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
const n = 20000
ms := make([]Message, n)
for i := range ms {
ms[i] = msg(fmt.Sprintf("<m%d@x>", i), "S", "b")
func TestThreadRootsIsLinearOnLongChains(t *testing.T) {
const n = 300_000
ids := make([]int64, n)
parent := make(map[int64]int64, n)
for i := range ids {
ids[i] = int64(i + 1)
if i > 0 {
ms[i].InReplyTo = fmt.Sprintf("<m%d@x>", i-1)
parent[int64(i+1)] = int64(i)
}
}
if _, err := s.ArchiveBatch(l.ID, ms); err != nil {
t.Fatal(err)
}
done := make(chan struct{})
go func() { s.Rethread(l.ID); close(done) }()
done := make(chan map[int64]int64, 1)
go func() { done <- threadRoots(ids, parent) }()
select {
case <-done:
case roots := <-done:
if roots[n] != 1 || roots[1] != 1 || roots[n/2] != 1 {
t.Fatalf("everything is in the thread of message 1: %d %d %d", roots[n], roots[1], roots[n/2])
}
case <-time.After(5 * time.Second):
t.Fatal("Rethread is not linear: a 20000-message chain took more than 5 seconds")
t.Fatal("threadRoots is not linear: a 300000-message chain took more than 5 seconds")
}
}
func TestThreadRootsFromTheNewestEndOfAChain(t *testing.T) {
// Visiting the newest message first must give the same roots as visiting the oldest first.
parent := map[int64]int64{2: 1, 3: 2, 4: 3, 6: 5}
forward := threadRoots([]int64{1, 2, 3, 4, 5, 6}, parent)
backward := threadRoots([]int64{6, 5, 4, 3, 2, 1}, parent)
for _, id := range []int64{1, 2, 3, 4, 5, 6} {
if forward[id] != backward[id] {
t.Fatalf("order dependent: %d -> %d vs %d", id, forward[id], backward[id])
}
}
if forward[4] != 1 || forward[6] != 5 || forward[5] != 5 {
t.Fatalf("roots: %v", forward)
}
}

View file

@ -45,12 +45,39 @@ func (s *Store) Enqueue(msg []byte, items []NewOutbound) error {
return err
}
defer tx.Rollback()
if err := enqueueTx(tx, s.Now().Unix(), msg, items); err != nil {
return err
}
return tx.Commit()
}
// EnqueueFromHeld releases a held message: it removes it from the moderation queue and queues
// its deliveries in one transaction. Of several simultaneous approvals only one succeeds, the
// others get ErrNotFound, so a message is never delivered twice. With no recipients the message
// is still released.
func (s *Store) EnqueueFromHeld(heldID int64, msg []byte, items []NewOutbound) error {
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback()
if err := affected(tx.Exec(`DELETE FROM held WHERE id=?`, heldID)); err != nil {
return err
}
if len(items) > 0 {
if err := enqueueTx(tx, s.Now().Unix(), msg, items); err != nil {
return err
}
}
return tx.Commit()
}
func enqueueTx(tx *sql.Tx, now int64, msg []byte, items []NewOutbound) error {
res, err := tx.Exec(`INSERT INTO outbox_msgs(data) VALUES(?)`, msg)
if err != nil {
return err
}
msgID, _ := res.LastInsertId()
now := s.Now().Unix()
for _, it := range items {
prefix := it.Prefix
if prefix == nil {
@ -61,7 +88,7 @@ func (s *Store) Enqueue(msg []byte, items []NewOutbound) error {
return err
}
}
return tx.Commit()
return nil
}
const outboxCols = `id, msg_id, envelope_from, rcpt, member_id, prefix, attempts, next_at, last_error, created_at`

View file

@ -188,3 +188,74 @@ func TestQueueSurvivesReopen(t *testing.T) {
t.Fatalf("data: %q", data)
}
}
func TestEnqueueFromHeldMovesTheMessageAtomically(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("Subject: s\r\n\r\nb"), "r")
if err := s.EnqueueFromHeld(hid, []byte("final"), items(2)); err != nil {
t.Fatal(err)
}
if _, err := s.GetHeld(hid); !errors.Is(err, ErrNotFound) {
t.Fatal("the held message is gone")
}
if st, _ := s.OutboxStats(); st.Pending != 2 {
t.Fatalf("pending = %d", st.Pending)
}
// Approving again must not queue anything.
if err := s.EnqueueFromHeld(hid, []byte("final"), items(2)); !errors.Is(err, ErrNotFound) {
t.Fatalf("want ErrNotFound, got %v", err)
}
if st, _ := s.OutboxStats(); st.Pending != 2 {
t.Fatalf("a second approval queued more mail: %d", st.Pending)
}
}
func TestEnqueueFromHeldWithNoRecipientsStillReleasesTheMessage(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r")
if err := s.EnqueueFromHeld(hid, []byte("final"), nil); err != nil {
t.Fatal(err)
}
if _, err := s.GetHeld(hid); !errors.Is(err, ErrNotFound) {
t.Fatal("the message is released even if nobody is subscribed")
}
if st, _ := s.OutboxStats(); st.Pending != 0 {
t.Fatalf("pending = %d", st.Pending)
}
}
func TestOnlyOneOfSeveralSimultaneousApprovalsWins(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r")
const n = 12
results := make(chan error, n)
start := make(chan struct{})
for i := 0; i < n; i++ {
go func() {
<-start
results <- s.EnqueueFromHeld(hid, []byte("final"), items(3))
}()
}
close(start)
wins, notFound := 0, 0
for i := 0; i < n; i++ {
switch err := <-results; {
case err == nil:
wins++
case errors.Is(err, ErrNotFound):
notFound++
default:
t.Fatalf("unexpected error: %v", err)
}
}
if wins != 1 || notFound != n-1 {
t.Fatalf("wins=%d notFound=%d", wins, notFound)
}
if st, _ := s.OutboxStats(); st.Pending != 3 {
t.Fatalf("the message must be queued exactly once for 3 recipients, pending = %d", st.Pending)
}
}

View file

@ -709,6 +709,47 @@ func archiveOne(q querier, listID int64, m Message) (id int64, created bool, err
return id, true, nil
}
// threadRoots maps every id to the root of its reply chain. parent[id] is the message id replies to,
// when that message exists. A cycle of replies has the smallest id of the cycle as root. Each chain
// is walked once, so the cost is linear in the number of messages.
func threadRoots(ids []int64, parent map[int64]int64) map[int64]int64 {
roots := make(map[int64]int64, len(ids))
for _, start := range ids {
if _, done := roots[start]; done {
continue
}
path := []int64{start}
seen := map[int64]int{start: 0}
cur := start
var result int64
for {
p, ok := parent[cur]
if !ok {
result = cur
break
}
if r, known := roots[p]; known {
result = r
break
}
if at, loop := seen[p]; loop { // a cycle: its smallest id is the root
result = path[at]
for _, id := range path[at:] {
result = min(result, id)
}
break
}
seen[p] = len(path)
path = append(path, p)
cur = p
}
for _, id := range path {
roots[id] = result
}
}
return roots
}
// Rethread recomputes the thread of every message of a list from the In-Reply-To
// links, and returns how many messages changed thread. Insertion order does not
// matter: a message imported before its parent ends up in the parent's thread.
@ -740,46 +781,14 @@ func (s *Store) Rethread(listID int64) (int, error) {
}
parent := map[int64]int64{}
ids := make([]int64, 0, len(all))
for _, r := range all {
ids = append(ids, r.id)
if p, ok := byMsgID[r.replyTo]; ok && r.replyTo != "" {
parent[r.id] = p
}
}
roots := map[int64]int64{}
root := func(start int64) int64 {
if r, ok := roots[start]; ok {
return r
}
path := []int64{start}
seen := map[int64]int{start: 0}
cur := start
var result int64
for {
p, ok := parent[cur]
if !ok {
result = cur
break
}
if r, known := roots[p]; known {
result = r
break
}
if at, loop := seen[p]; loop { // a cycle: its smallest id is the root
result = path[at]
for _, id := range path[at:] {
result = min(result, id)
}
break
}
seen[p] = len(path)
path = append(path, p)
cur = p
}
for _, id := range path {
roots[id] = result
}
return result
}
roots := threadRoots(ids, parent)
tx, err := s.db.Begin()
if err != nil {
@ -788,7 +797,7 @@ func (s *Store) Rethread(listID int64) (int, error) {
defer tx.Rollback()
changed := 0
for _, r := range all {
if want := root(r.id); want != r.thread {
if want := roots[r.id]; want != r.thread {
if _, err := tx.Exec(`UPDATE archive SET thread_id=? WHERE id=?`, want, r.id); err != nil {
return 0, err
}