// Package queue delivers the outbox to the SMTP relay in the background. package queue import ( "context" "errors" "fmt" "log" "sync" "time" "codeberg.org/squidd/parsimony/internal/outbound" "codeberg.org/squidd/parsimony/internal/store" ) // MaxAge is how long a failing delivery is retried before it is dropped. const MaxAge = 72 * time.Hour // Defaults for the zero values of Worker fields. const ( defaultConcurrency = 4 defaultBatch = 50 defaultTimeout = time.Minute defaultInterval = 5 * time.Second // purgeEvery is the least time between two scans that forget old failed deliveries. purgeEvery = 5 * time.Minute ) var backoff = []time.Duration{time.Minute, 5 * time.Minute, 15 * time.Minute, time.Hour, 3 * time.Hour, 6 * time.Hour} // Backoff returns how long to wait after the n-th consecutive failure (n >= 1). func Backoff(failures int) time.Duration { if failures < 1 { failures = 1 } if failures > len(backoff) { failures = len(backoff) } return backoff[failures-1] } // Stats counts what one run did. type Stats struct { Sent int // delivered to the relay Retried int // failed temporarily, scheduled again Dropped int // removed: the mailbox does not exist Failed int // given up for good (refused by the relay, or too old): kept visible for a week Bounced int // dropped deliveries that counted as a hard bounce } // Worker sends queued deliveries. Zero values of the optional fields mean defaults. type Worker struct { Store *store.Store Sender outbound.Sender Concurrency int // simultaneous sends Batch int // deliveries taken per run Timeout time.Duration // per send Interval time.Duration // idle polling, to pick up retries that have become due // Sign, when set, signs the final message (DKIM). On error the message is sent unsigned: // not delivering is worse than a missed signature. Sign func(msg []byte) ([]byte, error) wakeOnce sync.Once wake chan struct{} runMu sync.Mutex lastPurge time.Time // under runMu; zero until the first purge // Hooks for tests; nil means the Store methods. purge func() (int, error) readBody func(msgID int64) ([]byte, error) } // bodies reads each message body once per run, however many recipients share it. type bodies struct { read func(msgID int64) ([]byte, error) mu sync.Mutex m map[int64]*body } type body struct { once sync.Once data []byte err error } func (b *bodies) get(msgID int64) ([]byte, error) { b.mu.Lock() e := b.m[msgID] if e == nil { e = &body{} b.m[msgID] = e } b.mu.Unlock() e.once.Do(func() { e.data, e.err = b.read(msgID) }) return e.data, e.err } // purgeOld forgets old failed deliveries, at most once every purgeEvery. func (w *Worker) purgeOld() { now := w.Store.Now() if !w.lastPurge.IsZero() && now.Sub(w.lastPurge) < purgeEvery { return } purge := w.purge if purge == nil { purge = w.Store.PurgeFailedOutbox } if _, err := purge(); err != nil { log.Printf("queue: purge old failures: %v", err) return // try again on the next run } w.lastPurge = now } func (w *Worker) wakeCh() chan struct{} { w.wakeOnce.Do(func() { w.wake = make(chan struct{}, 1) }) return w.wake } // Wake asks Run to look at the queue now. It never blocks. func (w *Worker) Wake() { select { case w.wakeCh() <- struct{}{}: default: } } // Run delivers until ctx is cancelled. func (w *Worker) Run(ctx context.Context) { interval := w.Interval if interval <= 0 { interval = defaultInterval } timer := time.NewTimer(0) defer timer.Stop() for { select { case <-ctx.Done(): return case <-w.wakeCh(): case <-timer.C: } // Keep going while there is work, so a big post drains without waiting for the timer. for ctx.Err() == nil { st, err := w.RunOnce(ctx) if err != nil { log.Printf("queue: %v", err) break } if st == (Stats{}) { break } } if !timer.Stop() { select { case <-timer.C: default: } } timer.Reset(interval) } } // RunOnce attempts every delivery that is due, up to Batch, with bounded concurrency. func (w *Worker) RunOnce(ctx context.Context) (Stats, error) { w.runMu.Lock() defer w.runMu.Unlock() batch := w.Batch if batch <= 0 { batch = defaultBatch } conc := w.Concurrency if conc <= 0 { conc = defaultConcurrency } w.purgeOld() due, err := w.Store.DueOutbox(batch) if err != nil { return Stats{}, err } read := w.readBody if read == nil { read = w.Store.OutboxData } cache := &bodies{read: read, m: map[int64]*body{}} var ( mu sync.Mutex stats Stats wg sync.WaitGroup sem = make(chan struct{}, conc) ) for _, it := range due { if ctx.Err() != nil { break } sem <- struct{}{} wg.Add(1) go func(it store.OutboxItem) { defer wg.Done() defer func() { <-sem }() o := w.deliver(ctx, cache, it) mu.Lock() stats.Sent += o.Sent stats.Retried += o.Retried stats.Dropped += o.Dropped stats.Failed += o.Failed stats.Bounced += o.Bounced mu.Unlock() }(it) } wg.Wait() return stats, nil } func (w *Worker) deliver(ctx context.Context, cache *bodies, it store.OutboxItem) Stats { data, err := cache.get(it.MsgID) if errors.Is(err, store.ErrNotFound) { w.drop(it) return Stats{Dropped: 1} } if err != nil { log.Printf("queue: read message %d: %v", it.MsgID, err) return Stats{} } msg := make([]byte, 0, len(it.Prefix)+len(data)) msg = append(append(msg, it.Prefix...), data...) if w.Sign != nil { if signed, err := w.Sign(msg); err != nil { log.Printf("queue: sign message for %s: %v (sending unsigned)", it.Rcpt, err) } else { msg = signed } } timeout := w.Timeout if timeout <= 0 { timeout = defaultTimeout } sctx, cancel := context.WithTimeout(ctx, timeout) err = w.Sender.Send(sctx, it.From, []string{it.Rcpt}, msg) cancel() switch { case err == nil: w.complete(it) return Stats{Sent: 1} case ctx.Err() != nil: return Stats{} // shutting down: the delivery stays queued, unchanged case outbound.IsBadMailbox(err): // The address does not exist: nothing more to say about it, but count it against the member. out := Stats{Dropped: 1} if it.MemberID != 0 { if _, e := w.Store.RecordHardBounce(it.MemberID, store.BounceThreshold); e == nil { out.Bounced = 1 } } log.Printf("queue: drop %s: %v", it.Rcpt, err) w.drop(it) return out case outbound.IsPermanent(err): // The relay refused the message itself (policy, size, line length...): nobody gets it, so // keep the delivery listed with the reason instead of making it vanish. return w.giveUp(it, fmt.Sprintf("refused by the relay: %v", err)) } now := w.Store.Now() if now.Sub(it.CreatedAt) > MaxAge { return w.giveUp(it, fmt.Sprintf("gave up after %s: %v", MaxAge, err)) } if e := w.Store.RetryOutbox(it.ID, now.Add(Backoff(it.Attempts+1)), err.Error()); e != nil { log.Printf("queue: reschedule %d: %v", it.ID, e) return Stats{} } return Stats{Retried: 1} } // giveUp marks a delivery as failed for good and logs it. func (w *Worker) giveUp(it store.OutboxItem, reason string) Stats { log.Printf("queue: %s: %s", it.Rcpt, reason) if err := w.Store.FailOutbox(it.ID, reason); err != nil { log.Printf("queue: mark %d failed: %v", it.ID, err) return Stats{} } return Stats{Failed: 1} } func (w *Worker) complete(it store.OutboxItem) { if err := w.Store.CompleteOutbox(it.ID); err != nil && !errors.Is(err, store.ErrNotFound) { log.Printf("queue: remove %d: %v", it.ID, err) } } // drop removes a delivery that will never be made; a delivery of a post counts as dropped. func (w *Worker) drop(it store.OutboxItem) { if err := w.Store.DropOutbox(it.ID); err != nil && !errors.Is(err, store.ErrNotFound) { log.Printf("queue: drop %d: %v", it.ID, err) } }