perf(queue): purge old failures at most every 5 minutes and read each body once per run

- RunOnce purged failed deliveries (a full scan) on every batch of 50: the Worker now
  does it at most once every 5 minutes, by the store clock; a failed purge is retried
  on the next run.
- Every recipient re-read the whole message from SQLite: a per-run map msgID -> body,
  filled once per message even under concurrency, is dropped when the run ends.
- Tests count purges across batches and after the interval, and body reads per message
  (including a missing body) through unexported hooks.
This commit is contained in:
tgiachi 2026-11-10 11:51:26 +01:00
commit d9012cc13a
2 changed files with 170 additions and 6 deletions

View file

@ -22,6 +22,9 @@ const (
defaultBatch = 50
defaultTimeout = time.Minute
defaultInterval = 5 * time.Second
// purgeEvery is the least time between two scans that forget old failed deliveries.
purgeEvery = 5 * time.Minute
)
var backoff = []time.Duration{time.Minute, 5 * time.Minute, 15 * time.Minute, time.Hour, 3 * time.Hour, 6 * time.Hour}
@ -61,6 +64,53 @@ type Worker struct {
wakeOnce sync.Once
wake chan struct{}
runMu sync.Mutex
lastPurge time.Time // under runMu; zero until the first purge
// Hooks for tests; nil means the Store methods.
purge func() (int, error)
readBody func(msgID int64) ([]byte, error)
}
// bodies reads each message body once per run, however many recipients share it.
type bodies struct {
read func(msgID int64) ([]byte, error)
mu sync.Mutex
m map[int64]*body
}
type body struct {
once sync.Once
data []byte
err error
}
func (b *bodies) get(msgID int64) ([]byte, error) {
b.mu.Lock()
e := b.m[msgID]
if e == nil {
e = &body{}
b.m[msgID] = e
}
b.mu.Unlock()
e.once.Do(func() { e.data, e.err = b.read(msgID) })
return e.data, e.err
}
// purgeOld forgets old failed deliveries, at most once every purgeEvery.
func (w *Worker) purgeOld() {
now := w.Store.Now()
if !w.lastPurge.IsZero() && now.Sub(w.lastPurge) < purgeEvery {
return
}
purge := w.purge
if purge == nil {
purge = w.Store.PurgeFailedOutbox
}
if _, err := purge(); err != nil {
log.Printf("queue: purge old failures: %v", err)
return // try again on the next run
}
w.lastPurge = now
}
func (w *Worker) wakeCh() chan struct{} {
@ -125,14 +175,18 @@ func (w *Worker) RunOnce(ctx context.Context) (Stats, error) {
if conc <= 0 {
conc = defaultConcurrency
}
if _, err := w.Store.PurgeFailedOutbox(); err != nil {
log.Printf("queue: purge old failures: %v", err)
}
w.purgeOld()
due, err := w.Store.DueOutbox(batch)
if err != nil {
return Stats{}, err
}
read := w.readBody
if read == nil {
read = w.Store.OutboxData
}
cache := &bodies{read: read, m: map[int64]*body{}}
var (
mu sync.Mutex
stats Stats
@ -148,7 +202,7 @@ func (w *Worker) RunOnce(ctx context.Context) (Stats, error) {
go func(it store.OutboxItem) {
defer wg.Done()
defer func() { <-sem }()
o := w.deliver(ctx, it)
o := w.deliver(ctx, cache, it)
mu.Lock()
stats.Sent += o.Sent
stats.Retried += o.Retried
@ -162,8 +216,8 @@ func (w *Worker) RunOnce(ctx context.Context) (Stats, error) {
return stats, nil
}
func (w *Worker) deliver(ctx context.Context, it store.OutboxItem) Stats {
data, err := w.Store.OutboxData(it.MsgID)
func (w *Worker) deliver(ctx context.Context, cache *bodies, it store.OutboxItem) Stats {
data, err := cache.get(it.MsgID)
if errors.Is(err, store.ErrNotFound) {
w.complete(it)
return Stats{Dropped: 1}

View file

@ -558,3 +558,113 @@ func TestARefusedSenderIsNotAMissingMailbox(t *testing.T) {
t.Fatalf("every delivery stays listed as failed: %+v", stats)
}
}
func TestPurgeRunsOncePerIntervalNotOnEveryBatch(t *testing.T) {
e := setup(t)
var purges int32
e.w.purge = func() (int, error) { atomic.AddInt32(&purges, 1); return 0, nil }
e.w.Batch = 1
e.enqueue(t, 3)
for i := 0; i < 3; i++ {
if st, err := e.w.RunOnce(context.Background()); err != nil || st.Sent != 1 {
t.Fatalf("run %d: %+v %v", i, st, err)
}
}
if n := atomic.LoadInt32(&purges); n != 1 {
t.Fatalf("purge ran %d times in 3 batches, want 1", n)
}
e.clk.set(t0.Add(purgeEvery - time.Second))
e.w.RunOnce(context.Background())
if n := atomic.LoadInt32(&purges); n != 1 {
t.Fatalf("purge ran again before the interval: %d", n)
}
e.clk.set(t0.Add(purgeEvery))
e.w.RunOnce(context.Background())
if n := atomic.LoadInt32(&purges); n != 2 {
t.Fatalf("purge must run again after the interval: %d", n)
}
e.w.RunOnce(context.Background())
if n := atomic.LoadInt32(&purges); n != 2 {
t.Fatalf("interval counts from the last purge: %d", n)
}
}
func TestFailedPurgeIsRetriedOnTheNextRun(t *testing.T) {
e := setup(t)
var purges int32
e.w.purge = func() (int, error) { atomic.AddInt32(&purges, 1); return 0, errors.New("db busy") }
e.w.RunOnce(context.Background())
e.w.RunOnce(context.Background())
if n := atomic.LoadInt32(&purges); n != 2 {
t.Fatalf("a failed purge must not delay the next attempt: %d runs", n)
}
}
func TestOneBodyReadServesEveryRecipientOfAMessage(t *testing.T) {
e := setup(t)
reads := map[int64]int{}
var mu sync.Mutex
e.w.readBody = func(id int64) ([]byte, error) {
mu.Lock()
reads[id]++
mu.Unlock()
return e.s.OutboxData(id)
}
e.enqueue(t, 6)
e.w.Concurrency = 6
if st, err := e.w.RunOnce(context.Background()); err != nil || st.Sent != 6 {
t.Fatalf("run: %+v %v", st, err)
}
if len(reads) != 1 {
t.Fatalf("reads = %v, want one message", reads)
}
for id, n := range reads {
if n != 1 {
t.Fatalf("message %d read %d times, want 1", id, n)
}
}
for _, m := range e.snd.sent {
if !strings.HasSuffix(string(m.Data), "Subject: hi\r\n\r\nbody\r\n") {
t.Fatalf("data = %q", m.Data)
}
}
// The cache lives for one run only: a later run reads again.
e.enqueue(t, 1)
e.w.RunOnce(context.Background())
total := 0
for _, n := range reads {
total += n
}
if total != 2 {
t.Fatalf("the next run must read its own body: %v", reads)
}
}
func TestEachDistinctMessageIsReadOnce(t *testing.T) {
e := setup(t)
var calls int32
e.w.readBody = func(id int64) ([]byte, error) { atomic.AddInt32(&calls, 1); return e.s.OutboxData(id) }
for i := 0; i < 3; i++ {
e.enqueue(t, 2)
}
if st, _ := e.w.RunOnce(context.Background()); st.Sent != 6 {
t.Fatalf("sent %d", st.Sent)
}
if n := atomic.LoadInt32(&calls); n != 3 {
t.Fatalf("%d reads for 3 messages", n)
}
}
func TestAMissingBodyIsDroppedForEveryRecipientWithOneRead(t *testing.T) {
e := setup(t)
var calls int32
e.w.readBody = func(int64) ([]byte, error) { atomic.AddInt32(&calls, 1); return nil, store.ErrNotFound }
e.enqueue(t, 3)
st, _ := e.w.RunOnce(context.Background())
if st.Dropped != 3 || e.snd.count() != 0 {
t.Fatalf("stats %+v", st)
}
if n := atomic.LoadInt32(&calls); n != 1 {
t.Fatalf("%d reads", n)
}
}