feat(queue): background worker with retries, backoff and bounded concurrency
- Add Worker.RunOnce, Run and Wake over the durable outbox - Send prefix plus shared message per recipient with a per-send timeout - Retry temporary failures after 1m, 5m, 15m, 1h, 3h, then every 6h; give up after 3 days - Drop permanent failures; count a hard bounce only for bad-mailbox replies - Leave interrupted sends queued on shutdown - Share the bounce threshold between router and worker
This commit is contained in:
parent
ecbab4d18c
commit
e6713962f9
1 changed files with 638 additions and 4 deletions
211
internal/queue/queue.go
Normal file
211
internal/queue/queue.go
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
// Package queue delivers the outbox to the SMTP relay in the background.
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"codeberg.org/squidd/parsimony/internal/outbound"
|
||||
"codeberg.org/squidd/parsimony/internal/store"
|
||||
)
|
||||
|
||||
// MaxAge is how long a failing delivery is retried before it is dropped.
|
||||
const MaxAge = 72 * time.Hour
|
||||
|
||||
// Defaults for the zero values of Worker fields.
|
||||
const (
|
||||
defaultConcurrency = 4
|
||||
defaultBatch = 50
|
||||
defaultTimeout = time.Minute
|
||||
defaultInterval = 5 * time.Second
|
||||
)
|
||||
|
||||
var backoff = []time.Duration{time.Minute, 5 * time.Minute, 15 * time.Minute, time.Hour, 3 * time.Hour, 6 * time.Hour}
|
||||
|
||||
// Backoff returns how long to wait after the n-th consecutive failure (n >= 1).
|
||||
func Backoff(failures int) time.Duration {
|
||||
if failures < 1 {
|
||||
failures = 1
|
||||
}
|
||||
if failures > len(backoff) {
|
||||
failures = len(backoff)
|
||||
}
|
||||
return backoff[failures-1]
|
||||
}
|
||||
|
||||
// Stats counts what one run did.
|
||||
type Stats struct {
|
||||
Sent int // delivered to the relay
|
||||
Retried int // failed temporarily, scheduled again
|
||||
Dropped int // removed: permanent failure or too old
|
||||
Bounced int // dropped deliveries that counted as a hard bounce
|
||||
}
|
||||
|
||||
// Worker sends queued deliveries. Zero values of the optional fields mean defaults.
|
||||
type Worker struct {
|
||||
Store *store.Store
|
||||
Sender outbound.Sender
|
||||
Concurrency int // simultaneous sends
|
||||
Batch int // deliveries taken per run
|
||||
Timeout time.Duration // per send
|
||||
Interval time.Duration // idle polling, to pick up retries that have become due
|
||||
|
||||
wakeOnce sync.Once
|
||||
wake chan struct{}
|
||||
runMu sync.Mutex
|
||||
}
|
||||
|
||||
func (w *Worker) wakeCh() chan struct{} {
|
||||
w.wakeOnce.Do(func() { w.wake = make(chan struct{}, 1) })
|
||||
return w.wake
|
||||
}
|
||||
|
||||
// Wake asks Run to look at the queue now. It never blocks.
|
||||
func (w *Worker) Wake() {
|
||||
select {
|
||||
case w.wakeCh() <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// Run delivers until ctx is cancelled.
|
||||
func (w *Worker) Run(ctx context.Context) {
|
||||
interval := w.Interval
|
||||
if interval <= 0 {
|
||||
interval = defaultInterval
|
||||
}
|
||||
timer := time.NewTimer(0)
|
||||
defer timer.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-w.wakeCh():
|
||||
case <-timer.C:
|
||||
}
|
||||
// Keep going while there is work, so a big post drains without waiting for the timer.
|
||||
for ctx.Err() == nil {
|
||||
st, err := w.RunOnce(ctx)
|
||||
if err != nil {
|
||||
log.Printf("queue: %v", err)
|
||||
break
|
||||
}
|
||||
if st == (Stats{}) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if !timer.Stop() {
|
||||
select {
|
||||
case <-timer.C:
|
||||
default:
|
||||
}
|
||||
}
|
||||
timer.Reset(interval)
|
||||
}
|
||||
}
|
||||
|
||||
// RunOnce attempts every delivery that is due, up to Batch, with bounded concurrency.
|
||||
func (w *Worker) RunOnce(ctx context.Context) (Stats, error) {
|
||||
w.runMu.Lock()
|
||||
defer w.runMu.Unlock()
|
||||
|
||||
batch := w.Batch
|
||||
if batch <= 0 {
|
||||
batch = defaultBatch
|
||||
}
|
||||
conc := w.Concurrency
|
||||
if conc <= 0 {
|
||||
conc = defaultConcurrency
|
||||
}
|
||||
due, err := w.Store.DueOutbox(batch)
|
||||
if err != nil {
|
||||
return Stats{}, err
|
||||
}
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
stats Stats
|
||||
wg sync.WaitGroup
|
||||
sem = make(chan struct{}, conc)
|
||||
)
|
||||
for _, it := range due {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
sem <- struct{}{}
|
||||
wg.Add(1)
|
||||
go func(it store.OutboxItem) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
o := w.deliver(ctx, it)
|
||||
mu.Lock()
|
||||
stats.Sent += o.Sent
|
||||
stats.Retried += o.Retried
|
||||
stats.Dropped += o.Dropped
|
||||
stats.Bounced += o.Bounced
|
||||
mu.Unlock()
|
||||
}(it)
|
||||
}
|
||||
wg.Wait()
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func (w *Worker) deliver(ctx context.Context, it store.OutboxItem) Stats {
|
||||
data, err := w.Store.OutboxData(it.MsgID)
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
w.complete(it)
|
||||
return Stats{Dropped: 1}
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("queue: read message %d: %v", it.MsgID, err)
|
||||
return Stats{}
|
||||
}
|
||||
msg := make([]byte, 0, len(it.Prefix)+len(data))
|
||||
msg = append(append(msg, it.Prefix...), data...)
|
||||
|
||||
timeout := w.Timeout
|
||||
if timeout <= 0 {
|
||||
timeout = defaultTimeout
|
||||
}
|
||||
sctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
err = w.Sender.Send(sctx, it.From, []string{it.Rcpt}, msg)
|
||||
cancel()
|
||||
|
||||
switch {
|
||||
case err == nil:
|
||||
w.complete(it)
|
||||
return Stats{Sent: 1}
|
||||
case ctx.Err() != nil:
|
||||
return Stats{} // shutting down: the delivery stays queued, unchanged
|
||||
case outbound.IsPermanent(err):
|
||||
out := Stats{Dropped: 1}
|
||||
if outbound.IsBadMailbox(err) && it.MemberID != 0 {
|
||||
if _, e := w.Store.RecordHardBounce(it.MemberID, store.BounceThreshold); e == nil {
|
||||
out.Bounced = 1
|
||||
}
|
||||
}
|
||||
log.Printf("queue: drop %s: %v", it.Rcpt, err)
|
||||
w.complete(it)
|
||||
return out
|
||||
}
|
||||
now := w.Store.Now()
|
||||
if now.Sub(it.CreatedAt) > MaxAge {
|
||||
log.Printf("queue: give up on %s after %s: %v", it.Rcpt, MaxAge, err)
|
||||
w.complete(it)
|
||||
return Stats{Dropped: 1}
|
||||
}
|
||||
if e := w.Store.RetryOutbox(it.ID, now.Add(Backoff(it.Attempts+1)), err.Error()); e != nil {
|
||||
log.Printf("queue: reschedule %d: %v", it.ID, e)
|
||||
return Stats{}
|
||||
}
|
||||
return Stats{Retried: 1}
|
||||
}
|
||||
|
||||
func (w *Worker) complete(it store.OutboxItem) {
|
||||
if err := w.Store.CompleteOutbox(it.ID); err != nil && !errors.Is(err, store.ErrNotFound) {
|
||||
log.Printf("queue: remove %d: %v", it.ID, err)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue