fix: keep refused deliveries visible instead of dropping them silently

- Add failed deliveries: a permanent refusal by the relay (policy, size, line length) or the 3-day give-up marks the delivery failed, with the reason, and never retries it
- Keep failed deliveries listed for a week, then purge them and their stored message
- A non-existent mailbox still drops the delivery and counts a hard bounce
- Migrate the outbox table
This commit is contained in:
tgiachi 2026-11-09 20:33:27 +01:00
commit bb0858ef87
5 changed files with 211 additions and 23 deletions

View file

@ -4,6 +4,7 @@ package queue
import (
"context"
"errors"
"fmt"
"log"
"sync"
"time"
@ -40,7 +41,8 @@ func Backoff(failures int) time.Duration {
type Stats struct {
Sent int // delivered to the relay
Retried int // failed temporarily, scheduled again
Dropped int // removed: permanent failure or too old
Dropped int // removed: the mailbox does not exist
Failed int // given up for good (refused by the relay, or too old): kept visible for a week
Bounced int // dropped deliveries that counted as a hard bounce
}
@ -123,6 +125,9 @@ func (w *Worker) RunOnce(ctx context.Context) (Stats, error) {
if conc <= 0 {
conc = defaultConcurrency
}
if _, err := w.Store.PurgeFailedOutbox(); err != nil {
log.Printf("queue: purge old failures: %v", err)
}
due, err := w.Store.DueOutbox(batch)
if err != nil {
return Stats{}, err
@ -148,6 +153,7 @@ func (w *Worker) RunOnce(ctx context.Context) (Stats, error) {
stats.Sent += o.Sent
stats.Retried += o.Retried
stats.Dropped += o.Dropped
stats.Failed += o.Failed
stats.Bounced += o.Bounced
mu.Unlock()
}(it)
@ -190,9 +196,10 @@ func (w *Worker) deliver(ctx context.Context, it store.OutboxItem) Stats {
return Stats{Sent: 1}
case ctx.Err() != nil:
return Stats{} // shutting down: the delivery stays queued, unchanged
case outbound.IsPermanent(err):
case outbound.IsBadMailbox(err):
// The address does not exist: nothing more to say about it, but count it against the member.
out := Stats{Dropped: 1}
if outbound.IsBadMailbox(err) && it.MemberID != 0 {
if it.MemberID != 0 {
if _, e := w.Store.RecordHardBounce(it.MemberID, store.BounceThreshold); e == nil {
out.Bounced = 1
}
@ -200,12 +207,14 @@ func (w *Worker) deliver(ctx context.Context, it store.OutboxItem) Stats {
log.Printf("queue: drop %s: %v", it.Rcpt, err)
w.complete(it)
return out
case outbound.IsPermanent(err):
// The relay refused the message itself (policy, size, line length...): nobody gets it, so
// keep the delivery listed with the reason instead of making it vanish.
return w.giveUp(it, fmt.Sprintf("refused by the relay: %v", err))
}
now := w.Store.Now()
if now.Sub(it.CreatedAt) > MaxAge {
log.Printf("queue: give up on %s after %s: %v", it.Rcpt, MaxAge, err)
w.complete(it)
return Stats{Dropped: 1}
return w.giveUp(it, fmt.Sprintf("gave up after %s: %v", MaxAge, err))
}
if e := w.Store.RetryOutbox(it.ID, now.Add(Backoff(it.Attempts+1)), err.Error()); e != nil {
log.Printf("queue: reschedule %d: %v", it.ID, e)
@ -214,6 +223,16 @@ func (w *Worker) deliver(ctx context.Context, it store.OutboxItem) Stats {
return Stats{Retried: 1}
}
// giveUp marks a delivery as failed for good and logs it.
func (w *Worker) giveUp(it store.OutboxItem, reason string) Stats {
log.Printf("queue: %s: %s", it.Rcpt, reason)
if err := w.Store.FailOutbox(it.ID, reason); err != nil {
log.Printf("queue: mark %d failed: %v", it.ID, err)
return Stats{}
}
return Stats{Failed: 1}
}
func (w *Worker) complete(it store.OutboxItem) {
if err := w.Store.CompleteOutbox(it.ID); err != nil && !errors.Is(err, store.ErrNotFound) {
log.Printf("queue: remove %d: %v", it.ID, err)

View file

@ -203,15 +203,44 @@ func TestOneFailureDoesNotBlockTheOthers(t *testing.T) {
}
}
func TestPermanentFailureDropsWithoutBouncing(t *testing.T) {
func TestPermanentFailureIsKeptVisibleAndNeverRetried(t *testing.T) {
e := setup(t)
e.enqueue(t, 1)
calls := 0
e.snd.fn = func(context.Context, string, []string) error {
calls++
return &smtp.SMTPError{Code: 554, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "relay access denied"}
}
st, _ := e.w.RunOnce(context.Background())
if st.Dropped != 1 || st.Bounced != 0 || e.pending(t) != 0 {
t.Fatalf("%+v pending=%d", st, e.pending(t))
if st.Failed != 1 || st.Dropped != 0 || st.Bounced != 0 {
t.Fatalf("%+v", st)
}
stats, _ := e.s.OutboxStats()
if stats.Pending != 0 || stats.Failed != 1 {
t.Fatalf("the delivery is not lost, it is listed as failed: %+v", stats)
}
list, _ := e.s.ListOutbox(10)
if len(list) != 1 || !list[0].Failed || !strings.Contains(list[0].LastError, "relay access denied") {
t.Fatalf("the admin sees why: %+v", list)
}
e.clk.set(t0.Add(48 * time.Hour))
e.w.RunOnce(context.Background())
if calls != 1 {
t.Fatalf("a refused delivery is never retried, relay called %d times", calls)
}
}
func TestFailedDeliveriesAreForgottenAfterAWeekByTheWorker(t *testing.T) {
e := setup(t)
e.enqueue(t, 1)
e.snd.fn = func(context.Context, string, []string) error {
return &smtp.SMTPError{Code: 554, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "no"}
}
e.w.RunOnce(context.Background())
e.clk.set(t0.Add(8 * 24 * time.Hour))
e.w.RunOnce(context.Background())
if stats, _ := e.s.OutboxStats(); stats.Failed != 0 {
t.Fatalf("an old failure is purged on the next run: %+v", stats)
}
}
@ -252,7 +281,7 @@ func TestOtherPermanentRejectionsNeverCountAgainstTheMember(t *testing.T) {
for i := 0; i < 5; i++ {
e.s.Enqueue([]byte("x"), []store.NewOutbound{{From: "b@x.org", Rcpt: "ok@x.com", MemberID: ms[0].ID}})
st, _ := e.w.RunOnce(context.Background())
if st.Dropped != 1 || st.Bounced != 0 {
if st.Failed != 1 || st.Bounced != 0 {
t.Fatalf("round %d: %+v", i, st)
}
}
@ -288,9 +317,13 @@ func TestGivesUpAfterMaxAge(t *testing.T) {
// The retry delay may not have passed yet; move far enough.
e.clk.set(e.clk.now().Add(7 * time.Hour))
st, _ := e.w.RunOnce(context.Background())
if st.Dropped != 1 || e.pending(t) != 0 {
if st.Failed != 1 || e.pending(t) != 0 {
t.Fatalf("too old: %+v pending=%d", st, e.pending(t))
}
list, _ := e.s.ListOutbox(10)
if len(list) != 1 || !list[0].Failed || !strings.Contains(list[0].LastError, "gave up") || !strings.Contains(list[0].LastError, "still down") {
t.Fatalf("it stays listed with the reason: %+v", list)
}
}
func TestOldMailIsStillDeliveredIfTheRelayWorks(t *testing.T) {

View file

@ -26,15 +26,22 @@ type OutboxItem struct {
NextAt time.Time
LastError string
CreatedAt time.Time
// Failed deliveries are never retried (the relay refused them for good, or they are too old).
// They stay listed for a week so the admin can see what was lost and why.
Failed bool
}
// OutboxStats summarises the queue.
type OutboxStats struct {
Pending int // every queued delivery
Retrying int // deliveries that already failed at least once
Oldest time.Time // zero when the queue is empty
Pending int // deliveries still waiting to be sent
Retrying int // of those, the ones that already failed at least once
Failed int // given up: kept for a week for the admin to see
Oldest time.Time // oldest waiting delivery; zero when none waits
}
// failedTTL is how long a failed delivery stays listed.
const failedTTL = 7 * 24 * time.Hour
// Enqueue stores a message and one delivery per recipient, all or nothing.
func (s *Store) Enqueue(msg []byte, items []NewOutbound) error {
if len(items) == 0 {
@ -91,13 +98,13 @@ func enqueueTx(tx *sql.Tx, now int64, msg []byte, items []NewOutbound) error {
return nil
}
const outboxCols = `id, msg_id, envelope_from, rcpt, member_id, prefix, attempts, next_at, last_error, created_at`
const outboxCols = `id, msg_id, envelope_from, rcpt, member_id, prefix, attempts, next_at, last_error, created_at, failed`
func scanOutbox(row interface{ Scan(...any) error }) (OutboxItem, error) {
var it OutboxItem
var next, created int64
err := row.Scan(&it.ID, &it.MsgID, &it.From, &it.Rcpt, &it.MemberID, &it.Prefix, &it.Attempts, &next, &it.LastError, &created)
it.NextAt, it.CreatedAt = time.Unix(next, 0), time.Unix(created, 0)
var next, created, failed int64
err := row.Scan(&it.ID, &it.MsgID, &it.From, &it.Rcpt, &it.MemberID, &it.Prefix, &it.Attempts, &next, &it.LastError, &created, &failed)
it.NextAt, it.CreatedAt, it.Failed = time.Unix(next, 0), time.Unix(created, 0), failed != 0
return it, err
}
@ -120,13 +127,13 @@ func (s *Store) queryOutbox(q string, args ...any) ([]OutboxItem, error) {
// DueOutbox returns up to limit deliveries whose time has come, longest waiting first.
func (s *Store) DueOutbox(limit int) ([]OutboxItem, error) {
return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox WHERE next_at<=? ORDER BY next_at, id LIMIT ?`,
return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox WHERE failed=0 AND next_at<=? ORDER BY next_at, id LIMIT ?`,
s.Now().Unix(), limit)
}
// ListOutbox returns up to limit deliveries for display, failing ones first.
func (s *Store) ListOutbox(limit int) ([]OutboxItem, error) {
return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox ORDER BY attempts DESC, next_at, id LIMIT ?`, limit)
return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox ORDER BY failed DESC, attempts DESC, next_at, id LIMIT ?`, limit)
}
// OutboxData returns the shared message of a queued delivery.
@ -174,10 +181,38 @@ func (s *Store) RetryOutbox(id int64, next time.Time, lastError string) error {
func (s *Store) OutboxStats() (OutboxStats, error) {
var st OutboxStats
var oldest sql.NullInt64
err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(SUM(attempts>0),0), MIN(created_at) FROM outbox`).
Scan(&st.Pending, &st.Retrying, &oldest)
err := s.db.QueryRow(`SELECT COALESCE(SUM(failed=0),0), COALESCE(SUM(failed=0 AND attempts>0),0), COALESCE(SUM(failed),0),
MIN(CASE WHEN failed=0 THEN created_at END) FROM outbox`).Scan(&st.Pending, &st.Retrying, &st.Failed, &oldest)
if oldest.Valid {
st.Oldest = time.Unix(oldest.Int64, 0)
}
return st, err
}
// FailOutbox gives up on a delivery for good, keeping it listed with the reason. Its time is the
// moment of the failure, from which PurgeFailedOutbox counts.
func (s *Store) FailOutbox(id int64, reason string) error {
return affected(s.db.Exec(`UPDATE outbox SET failed=1, attempts=attempts+1, last_error=?, next_at=? WHERE id=?`,
reason, s.Now().Unix(), id))
}
// PurgeFailedOutbox forgets failed deliveries older than a week, and the stored messages nobody
// needs any more. It returns how many deliveries were removed.
func (s *Store) PurgeFailedOutbox() (int, error) {
tx, err := s.db.Begin()
if err != nil {
return 0, err
}
defer tx.Rollback()
res, err := tx.Exec(`DELETE FROM outbox WHERE failed=1 AND next_at<?`, s.Now().Add(-failedTTL).Unix())
if err != nil {
return 0, err
}
n, _ := res.RowsAffected()
if n > 0 {
if _, err := tx.Exec(`DELETE FROM outbox_msgs WHERE id NOT IN (SELECT msg_id FROM outbox)`); err != nil {
return 0, err
}
}
return int(n), tx.Commit()
}

View file

@ -259,3 +259,94 @@ func TestOnlyOneOfSeveralSimultaneousApprovalsWins(t *testing.T) {
t.Fatalf("the message must be queued exactly once for 3 recipients, pending = %d", st.Pending)
}
}
func TestFailedDeliveriesStayVisibleButAreNeverRetried(t *testing.T) {
s := newTestStore(t)
now := time.Unix(1_000_000, 0)
s.Now = func() time.Time { return now }
s.Enqueue([]byte("body"), items(3))
due, _ := s.DueOutbox(10)
if err := s.FailOutbox(due[0].ID, "relay said: 554 policy rejection"); err != nil {
t.Fatal(err)
}
if err := s.FailOutbox(999, "x"); !errors.Is(err, ErrNotFound) {
t.Fatalf("want ErrNotFound, got %v", err)
}
got, _ := s.DueOutbox(10)
if len(got) != 2 {
t.Fatalf("a failed delivery is not due again: %d due", len(got))
}
now = now.Add(24 * time.Hour)
if got, _ := s.DueOutbox(10); len(got) != 2 {
t.Fatal("not even a day later")
}
st, _ := s.OutboxStats()
if st.Pending != 2 || st.Failed != 1 || st.Retrying != 0 {
t.Fatalf("stats: %+v", st)
}
list, _ := s.ListOutbox(10)
if len(list) != 3 || !list[0].Failed || list[0].ID != due[0].ID || list[0].LastError != "relay said: 554 policy rejection" {
t.Fatalf("failed deliveries come first, with the reason: %+v", list)
}
// The admin can discard it, like any other.
if err := s.CompleteOutbox(due[0].ID); err != nil {
t.Fatal(err)
}
if st, _ := s.OutboxStats(); st.Failed != 0 {
t.Fatalf("%+v", st)
}
}
func TestFailedDeliveriesAreForgottenAfterAWeek(t *testing.T) {
s := newTestStore(t)
now := time.Unix(1_000_000, 0)
s.Now = func() time.Time { return now }
s.Enqueue([]byte("old body"), items(1))
s.Enqueue([]byte("new body"), items(1))
due, _ := s.DueOutbox(10)
s.FailOutbox(due[0].ID, "old failure")
now = now.Add(5 * 24 * time.Hour)
s.FailOutbox(due[1].ID, "recent failure")
now = now.Add(3 * 24 * time.Hour) // the first failure is now 8 days old, the second 3
n, err := s.PurgeFailedOutbox()
if err != nil || n != 1 {
t.Fatalf("purged %d: %v", n, err)
}
list, _ := s.ListOutbox(10)
if len(list) != 1 || list[0].LastError != "recent failure" {
t.Fatalf("list: %+v", list)
}
if _, err := s.OutboxData(due[0].MsgID); !errors.Is(err, ErrNotFound) {
t.Fatal("the stored message of a purged delivery goes too")
}
if _, err := s.OutboxData(due[1].MsgID); err != nil {
t.Fatalf("the recent one keeps its message: %v", err)
}
// Waiting and retrying deliveries are never purged, however old.
s.Enqueue([]byte("waiting"), items(1))
now = now.Add(365 * 24 * time.Hour)
s.PurgeFailedOutbox()
if st, _ := s.OutboxStats(); st.Pending != 1 {
t.Fatalf("a waiting delivery must survive: %+v", st)
}
}
func TestMigrationAddsTheFailedColumn(t *testing.T) {
path := t.TempDir() + "/old.db"
s, _ := Open(path)
s.Enqueue([]byte("x"), items(1))
s.db.Exec(`ALTER TABLE outbox DROP COLUMN failed`)
s.Close()
s2, err := Open(path)
if err != nil {
t.Fatalf("migrate: %v", err)
}
defer s2.Close()
due, _ := s2.DueOutbox(10)
if len(due) != 1 || due[0].Failed {
t.Fatalf("an existing delivery stays waiting: %+v", due)
}
}

View file

@ -184,7 +184,8 @@ CREATE TABLE IF NOT EXISTS outbox (
attempts INTEGER NOT NULL DEFAULT 0,
next_at INTEGER NOT NULL,
last_error TEXT NOT NULL DEFAULT '',
created_at INTEGER NOT NULL
created_at INTEGER NOT NULL,
failed INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS outbox_due ON outbox(next_at, id);
CREATE TABLE IF NOT EXISTS settings (
@ -257,6 +258,15 @@ func migrate(db *sql.DB) error {
}
}
}
if ok, err := hasColumn(db, "outbox", "msg_id"); err != nil {
return err
} else if ok {
if has, _ := hasColumn(db, "outbox", "failed"); !has {
if _, err := db.Exec(`ALTER TABLE outbox ADD COLUMN failed INTEGER NOT NULL DEFAULT 0`); err != nil {
return err
}
}
}
if ok, err := hasColumn(db, "held", "list_id"); err != nil {
return err
} else if ok {