diff --git a/internal/queue/queue.go b/internal/queue/queue.go index 201ddf2..9976a9c 100644 --- a/internal/queue/queue.go +++ b/internal/queue/queue.go @@ -22,6 +22,9 @@ const ( defaultBatch = 50 defaultTimeout = time.Minute defaultInterval = 5 * time.Second + + // purgeEvery is the least time between two scans that forget old failed deliveries. + purgeEvery = 5 * time.Minute ) var backoff = []time.Duration{time.Minute, 5 * time.Minute, 15 * time.Minute, time.Hour, 3 * time.Hour, 6 * time.Hour} @@ -61,6 +64,53 @@ type Worker struct { wakeOnce sync.Once wake chan struct{} runMu sync.Mutex + + lastPurge time.Time // under runMu; zero until the first purge + // Hooks for tests; nil means the Store methods. + purge func() (int, error) + readBody func(msgID int64) ([]byte, error) +} + +// bodies reads each message body once per run, however many recipients share it. +type bodies struct { + read func(msgID int64) ([]byte, error) + mu sync.Mutex + m map[int64]*body +} + +type body struct { + once sync.Once + data []byte + err error +} + +func (b *bodies) get(msgID int64) ([]byte, error) { + b.mu.Lock() + e := b.m[msgID] + if e == nil { + e = &body{} + b.m[msgID] = e + } + b.mu.Unlock() + e.once.Do(func() { e.data, e.err = b.read(msgID) }) + return e.data, e.err +} + +// purgeOld forgets old failed deliveries, at most once every purgeEvery. +func (w *Worker) purgeOld() { + now := w.Store.Now() + if !w.lastPurge.IsZero() && now.Sub(w.lastPurge) < purgeEvery { + return + } + purge := w.purge + if purge == nil { + purge = w.Store.PurgeFailedOutbox + } + if _, err := purge(); err != nil { + log.Printf("queue: purge old failures: %v", err) + return // try again on the next run + } + w.lastPurge = now } func (w *Worker) wakeCh() chan struct{} { @@ -125,14 +175,18 @@ func (w *Worker) RunOnce(ctx context.Context) (Stats, error) { if conc <= 0 { conc = defaultConcurrency } - if _, err := w.Store.PurgeFailedOutbox(); err != nil { - log.Printf("queue: purge old failures: %v", err) - } + w.purgeOld() due, err := w.Store.DueOutbox(batch) if err != nil { return Stats{}, err } + read := w.readBody + if read == nil { + read = w.Store.OutboxData + } + cache := &bodies{read: read, m: map[int64]*body{}} + var ( mu sync.Mutex stats Stats @@ -148,7 +202,7 @@ func (w *Worker) RunOnce(ctx context.Context) (Stats, error) { go func(it store.OutboxItem) { defer wg.Done() defer func() { <-sem }() - o := w.deliver(ctx, it) + o := w.deliver(ctx, cache, it) mu.Lock() stats.Sent += o.Sent stats.Retried += o.Retried @@ -162,8 +216,8 @@ func (w *Worker) RunOnce(ctx context.Context) (Stats, error) { return stats, nil } -func (w *Worker) deliver(ctx context.Context, it store.OutboxItem) Stats { - data, err := w.Store.OutboxData(it.MsgID) +func (w *Worker) deliver(ctx context.Context, cache *bodies, it store.OutboxItem) Stats { + data, err := cache.get(it.MsgID) if errors.Is(err, store.ErrNotFound) { w.complete(it) return Stats{Dropped: 1} diff --git a/internal/queue/queue_test.go b/internal/queue/queue_test.go index 7cd9f76..7e2dbd2 100644 --- a/internal/queue/queue_test.go +++ b/internal/queue/queue_test.go @@ -558,3 +558,113 @@ func TestARefusedSenderIsNotAMissingMailbox(t *testing.T) { t.Fatalf("every delivery stays listed as failed: %+v", stats) } } + +func TestPurgeRunsOncePerIntervalNotOnEveryBatch(t *testing.T) { + e := setup(t) + var purges int32 + e.w.purge = func() (int, error) { atomic.AddInt32(&purges, 1); return 0, nil } + e.w.Batch = 1 + e.enqueue(t, 3) + for i := 0; i < 3; i++ { + if st, err := e.w.RunOnce(context.Background()); err != nil || st.Sent != 1 { + t.Fatalf("run %d: %+v %v", i, st, err) + } + } + if n := atomic.LoadInt32(&purges); n != 1 { + t.Fatalf("purge ran %d times in 3 batches, want 1", n) + } + e.clk.set(t0.Add(purgeEvery - time.Second)) + e.w.RunOnce(context.Background()) + if n := atomic.LoadInt32(&purges); n != 1 { + t.Fatalf("purge ran again before the interval: %d", n) + } + e.clk.set(t0.Add(purgeEvery)) + e.w.RunOnce(context.Background()) + if n := atomic.LoadInt32(&purges); n != 2 { + t.Fatalf("purge must run again after the interval: %d", n) + } + e.w.RunOnce(context.Background()) + if n := atomic.LoadInt32(&purges); n != 2 { + t.Fatalf("interval counts from the last purge: %d", n) + } +} + +func TestFailedPurgeIsRetriedOnTheNextRun(t *testing.T) { + e := setup(t) + var purges int32 + e.w.purge = func() (int, error) { atomic.AddInt32(&purges, 1); return 0, errors.New("db busy") } + e.w.RunOnce(context.Background()) + e.w.RunOnce(context.Background()) + if n := atomic.LoadInt32(&purges); n != 2 { + t.Fatalf("a failed purge must not delay the next attempt: %d runs", n) + } +} + +func TestOneBodyReadServesEveryRecipientOfAMessage(t *testing.T) { + e := setup(t) + reads := map[int64]int{} + var mu sync.Mutex + e.w.readBody = func(id int64) ([]byte, error) { + mu.Lock() + reads[id]++ + mu.Unlock() + return e.s.OutboxData(id) + } + e.enqueue(t, 6) + e.w.Concurrency = 6 + if st, err := e.w.RunOnce(context.Background()); err != nil || st.Sent != 6 { + t.Fatalf("run: %+v %v", st, err) + } + if len(reads) != 1 { + t.Fatalf("reads = %v, want one message", reads) + } + for id, n := range reads { + if n != 1 { + t.Fatalf("message %d read %d times, want 1", id, n) + } + } + for _, m := range e.snd.sent { + if !strings.HasSuffix(string(m.Data), "Subject: hi\r\n\r\nbody\r\n") { + t.Fatalf("data = %q", m.Data) + } + } + // The cache lives for one run only: a later run reads again. + e.enqueue(t, 1) + e.w.RunOnce(context.Background()) + total := 0 + for _, n := range reads { + total += n + } + if total != 2 { + t.Fatalf("the next run must read its own body: %v", reads) + } +} + +func TestEachDistinctMessageIsReadOnce(t *testing.T) { + e := setup(t) + var calls int32 + e.w.readBody = func(id int64) ([]byte, error) { atomic.AddInt32(&calls, 1); return e.s.OutboxData(id) } + for i := 0; i < 3; i++ { + e.enqueue(t, 2) + } + if st, _ := e.w.RunOnce(context.Background()); st.Sent != 6 { + t.Fatalf("sent %d", st.Sent) + } + if n := atomic.LoadInt32(&calls); n != 3 { + t.Fatalf("%d reads for 3 messages", n) + } +} + +func TestAMissingBodyIsDroppedForEveryRecipientWithOneRead(t *testing.T) { + e := setup(t) + var calls int32 + e.w.readBody = func(int64) ([]byte, error) { atomic.AddInt32(&calls, 1); return nil, store.ErrNotFound } + e.enqueue(t, 3) + st, _ := e.w.RunOnce(context.Background()) + if st.Dropped != 3 || e.snd.count() != 0 { + t.Fatalf("stats %+v", st) + } + if n := atomic.LoadInt32(&calls); n != 1 { + t.Fatalf("%d reads", n) + } +}