Worker.Timeout defaulted to one minute over the entire SMTP transaction, cutting relays that are slow to accept a message and causing duplicate deliveries. The default is now no total deadline: SMTPRelay bounds each phase itself. A positive Timeout still applies.
329 lines
8.7 KiB
Go
329 lines
8.7 KiB
Go
// Package queue delivers the outbox to the SMTP relay in the background.
|
|
package queue
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"sync"
|
|
"time"
|
|
|
|
"codeberg.org/squidd/parsimony/internal/outbound"
|
|
"codeberg.org/squidd/parsimony/internal/store"
|
|
)
|
|
|
|
// MaxAge is how long a failing delivery is retried before it is dropped.
|
|
const MaxAge = 72 * time.Hour
|
|
|
|
// Defaults for the zero values of Worker fields.
|
|
const (
|
|
defaultConcurrency = 4
|
|
defaultBatch = 50
|
|
defaultInterval = 5 * time.Second
|
|
|
|
// purgeEvery is the least time between two scans that forget old failed deliveries.
|
|
purgeEvery = 5 * time.Minute
|
|
)
|
|
|
|
var backoff = []time.Duration{time.Minute, 5 * time.Minute, 15 * time.Minute, time.Hour, 3 * time.Hour, 6 * time.Hour}
|
|
|
|
// Backoff returns how long to wait after the n-th consecutive failure (n >= 1).
|
|
func Backoff(failures int) time.Duration {
|
|
if failures < 1 {
|
|
failures = 1
|
|
}
|
|
if failures > len(backoff) {
|
|
failures = len(backoff)
|
|
}
|
|
return backoff[failures-1]
|
|
}
|
|
|
|
// Stats counts what one run did.
|
|
type Stats struct {
|
|
Sent int // delivered to the relay
|
|
Retried int // failed temporarily, scheduled again
|
|
Dropped int // removed: the mailbox does not exist
|
|
Failed int // given up for good (refused by the relay, or too old): kept visible for a week
|
|
Bounced int // dropped deliveries that counted as a hard bounce
|
|
}
|
|
|
|
// Worker sends queued deliveries. Zero values of the optional fields mean defaults.
|
|
type Worker struct {
|
|
Store *store.Store
|
|
Sender outbound.Sender
|
|
Concurrency int // simultaneous sends
|
|
Batch int // deliveries taken per run
|
|
Timeout time.Duration // per send; zero leaves the deadlines to Sender (SMTPRelay bounds each SMTP phase)
|
|
Interval time.Duration // idle polling, to pick up retries that have become due
|
|
// Sign, when set, signs the final message (DKIM). On error the message is sent unsigned:
|
|
// not delivering is worse than a missed signature.
|
|
Sign func(msg []byte) ([]byte, error)
|
|
// Rate is the most sends started per second, for relays that throttle (Amazon SES). Zero or
|
|
// negative means unlimited. It is read once, at the first send.
|
|
Rate float64
|
|
|
|
pacerOnce sync.Once
|
|
pacer *pacer
|
|
// Hooks for tests; nil means the real clock and sleep.
|
|
pacerNow func() time.Time
|
|
pacerSleep func(ctx context.Context, d time.Duration) error
|
|
|
|
wakeOnce sync.Once
|
|
wake chan struct{}
|
|
runMu sync.Mutex
|
|
|
|
lastPurge time.Time // under runMu; zero until the first purge
|
|
// Hooks for tests; nil means the Store methods.
|
|
purge func() (int, error)
|
|
readBody func(msgID int64) ([]byte, error)
|
|
}
|
|
|
|
// bodies reads each message body once per run, however many recipients share it.
|
|
type bodies struct {
|
|
read func(msgID int64) ([]byte, error)
|
|
mu sync.Mutex
|
|
m map[int64]*body
|
|
}
|
|
|
|
type body struct {
|
|
once sync.Once
|
|
data []byte
|
|
err error
|
|
}
|
|
|
|
func (b *bodies) get(msgID int64) ([]byte, error) {
|
|
b.mu.Lock()
|
|
e := b.m[msgID]
|
|
if e == nil {
|
|
e = &body{}
|
|
b.m[msgID] = e
|
|
}
|
|
b.mu.Unlock()
|
|
e.once.Do(func() { e.data, e.err = b.read(msgID) })
|
|
return e.data, e.err
|
|
}
|
|
|
|
// purgeOld forgets old failed deliveries, at most once every purgeEvery.
|
|
func (w *Worker) purgeOld() {
|
|
now := w.Store.Now()
|
|
if !w.lastPurge.IsZero() && now.Sub(w.lastPurge) < purgeEvery {
|
|
return
|
|
}
|
|
purge := w.purge
|
|
if purge == nil {
|
|
purge = w.Store.PurgeFailedOutbox
|
|
}
|
|
if _, err := purge(); err != nil {
|
|
log.Printf("queue: purge old failures: %v", err)
|
|
return // try again on the next run
|
|
}
|
|
w.lastPurge = now
|
|
}
|
|
|
|
func (w *Worker) wakeCh() chan struct{} {
|
|
w.wakeOnce.Do(func() { w.wake = make(chan struct{}, 1) })
|
|
return w.wake
|
|
}
|
|
|
|
// Wake asks Run to look at the queue now. It never blocks.
|
|
func (w *Worker) Wake() {
|
|
select {
|
|
case w.wakeCh() <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
|
|
// Run delivers until ctx is cancelled.
|
|
func (w *Worker) Run(ctx context.Context) {
|
|
interval := w.Interval
|
|
if interval <= 0 {
|
|
interval = defaultInterval
|
|
}
|
|
timer := time.NewTimer(0)
|
|
defer timer.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-w.wakeCh():
|
|
case <-timer.C:
|
|
}
|
|
// Keep going while there is work, so a big post drains without waiting for the timer.
|
|
for ctx.Err() == nil {
|
|
st, err := w.RunOnce(ctx)
|
|
if err != nil {
|
|
log.Printf("queue: %v", err)
|
|
break
|
|
}
|
|
if st == (Stats{}) {
|
|
break
|
|
}
|
|
}
|
|
if !timer.Stop() {
|
|
select {
|
|
case <-timer.C:
|
|
default:
|
|
}
|
|
}
|
|
timer.Reset(interval)
|
|
}
|
|
}
|
|
|
|
// RunOnce attempts every delivery that is due, up to Batch, with bounded concurrency.
|
|
func (w *Worker) RunOnce(ctx context.Context) (Stats, error) {
|
|
w.runMu.Lock()
|
|
defer w.runMu.Unlock()
|
|
|
|
batch := w.Batch
|
|
if batch <= 0 {
|
|
batch = defaultBatch
|
|
}
|
|
conc := w.Concurrency
|
|
if conc <= 0 {
|
|
conc = defaultConcurrency
|
|
}
|
|
w.purgeOld()
|
|
due, err := w.Store.DueOutbox(batch)
|
|
if err != nil {
|
|
return Stats{}, err
|
|
}
|
|
|
|
read := w.readBody
|
|
if read == nil {
|
|
read = w.Store.OutboxData
|
|
}
|
|
cache := &bodies{read: read, m: map[int64]*body{}}
|
|
|
|
var (
|
|
mu sync.Mutex
|
|
stats Stats
|
|
wg sync.WaitGroup
|
|
sem = make(chan struct{}, conc)
|
|
)
|
|
for _, it := range due {
|
|
if ctx.Err() != nil {
|
|
break
|
|
}
|
|
sem <- struct{}{}
|
|
wg.Add(1)
|
|
go func(it store.OutboxItem) {
|
|
defer wg.Done()
|
|
defer func() { <-sem }()
|
|
o := w.deliver(ctx, cache, it)
|
|
mu.Lock()
|
|
stats.Sent += o.Sent
|
|
stats.Retried += o.Retried
|
|
stats.Dropped += o.Dropped
|
|
stats.Failed += o.Failed
|
|
stats.Bounced += o.Bounced
|
|
mu.Unlock()
|
|
}(it)
|
|
}
|
|
wg.Wait()
|
|
return stats, nil
|
|
}
|
|
|
|
func (w *Worker) deliver(ctx context.Context, cache *bodies, it store.OutboxItem) Stats {
|
|
data, err := cache.get(it.MsgID)
|
|
if errors.Is(err, store.ErrNotFound) {
|
|
w.drop(it)
|
|
return Stats{Dropped: 1}
|
|
}
|
|
if err != nil {
|
|
log.Printf("queue: read message %d: %v", it.MsgID, err)
|
|
return Stats{}
|
|
}
|
|
msg := make([]byte, 0, len(it.Prefix)+len(data))
|
|
msg = append(append(msg, it.Prefix...), data...)
|
|
if w.Sign != nil {
|
|
if signed, err := w.Sign(msg); err != nil {
|
|
log.Printf("queue: sign message for %s: %v (sending unsigned)", it.Rcpt, err)
|
|
} else {
|
|
msg = signed
|
|
}
|
|
}
|
|
|
|
// Wait for the slot before starting the timeout, so waiting does not eat into the send.
|
|
if err := w.pace().Wait(ctx); err != nil {
|
|
return Stats{} // shutting down: the delivery stays queued, unchanged
|
|
}
|
|
// No total deadline by default: one that covers the whole transaction cut relays that take
|
|
// minutes to accept a message, which was then retried and delivered twice.
|
|
sctx, cancel := ctx, context.CancelFunc(func() {})
|
|
if w.Timeout > 0 {
|
|
sctx, cancel = context.WithTimeout(ctx, w.Timeout)
|
|
}
|
|
err = w.Sender.Send(sctx, it.From, []string{it.Rcpt}, msg)
|
|
cancel()
|
|
|
|
switch {
|
|
case err == nil:
|
|
w.complete(it)
|
|
return Stats{Sent: 1}
|
|
case ctx.Err() != nil:
|
|
return Stats{} // shutting down: the delivery stays queued, unchanged
|
|
case outbound.IsBadMailbox(err):
|
|
// The address does not exist: nothing more to say about it, but count it against the member.
|
|
out := Stats{Dropped: 1}
|
|
if it.MemberID != 0 {
|
|
if _, e := w.Store.RecordHardBounce(it.MemberID, store.BounceThreshold); e == nil {
|
|
out.Bounced = 1
|
|
}
|
|
}
|
|
log.Printf("queue: drop %s: %v", it.Rcpt, err)
|
|
w.drop(it)
|
|
return out
|
|
case outbound.IsPermanent(err):
|
|
// The relay refused the message itself (policy, size, line length...): nobody gets it, so
|
|
// keep the delivery listed with the reason instead of making it vanish.
|
|
return w.giveUp(it, fmt.Sprintf("refused by the relay: %v", err))
|
|
}
|
|
now := w.Store.Now()
|
|
if now.Sub(it.CreatedAt) > MaxAge {
|
|
return w.giveUp(it, fmt.Sprintf("gave up after %s: %v", MaxAge, err))
|
|
}
|
|
if e := w.Store.RetryOutbox(it.ID, now.Add(Backoff(it.Attempts+1)), err.Error()); e != nil {
|
|
log.Printf("queue: reschedule %d: %v", it.ID, e)
|
|
return Stats{}
|
|
}
|
|
return Stats{Retried: 1}
|
|
}
|
|
|
|
// pace returns the pacer in front of every send; nil (never waits) when Rate is not set.
|
|
func (w *Worker) pace() *pacer {
|
|
w.pacerOnce.Do(func() {
|
|
now, sleep := w.pacerNow, w.pacerSleep
|
|
if now == nil {
|
|
now = time.Now
|
|
}
|
|
if sleep == nil {
|
|
sleep = sleepCtx
|
|
}
|
|
w.pacer = newPacer(w.Rate, now, sleep)
|
|
})
|
|
return w.pacer
|
|
}
|
|
|
|
// giveUp marks a delivery as failed for good and logs it.
|
|
func (w *Worker) giveUp(it store.OutboxItem, reason string) Stats {
|
|
log.Printf("queue: %s: %s", it.Rcpt, reason)
|
|
if err := w.Store.FailOutbox(it.ID, reason); err != nil {
|
|
log.Printf("queue: mark %d failed: %v", it.ID, err)
|
|
return Stats{}
|
|
}
|
|
return Stats{Failed: 1}
|
|
}
|
|
|
|
func (w *Worker) complete(it store.OutboxItem) {
|
|
if err := w.Store.CompleteOutbox(it.ID); err != nil && !errors.Is(err, store.ErrNotFound) {
|
|
log.Printf("queue: remove %d: %v", it.ID, err)
|
|
}
|
|
}
|
|
|
|
// drop removes a delivery that will never be made; a delivery of a post counts as dropped.
|
|
func (w *Worker) drop(it store.OutboxItem) {
|
|
if err := w.Store.DropOutbox(it.ID); err != nil && !errors.Is(err, store.ErrNotFound) {
|
|
log.Printf("queue: drop %d: %v", it.ID, err)
|
|
}
|
|
}
|