From bb0858ef87f31c9fb6d8d9c37dde3ba7a1f6bd35 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Mon, 9 Nov 2026 20:33:27 +0100 Subject: [PATCH] fix: keep refused deliveries visible instead of dropping them silently - Add failed deliveries: a permanent refusal by the relay (policy, size, line length) or the 3-day give-up marks the delivery failed, with the reason, and never retries it - Keep failed deliveries listed for a week, then purge them and their stored message - A non-existent mailbox still drops the delivery and counts a hard bounce - Migrate the outbox table --- internal/queue/queue.go | 31 +++++++++--- internal/queue/queue_test.go | 43 +++++++++++++++-- internal/store/outbox.go | 57 +++++++++++++++++----- internal/store/outbox_test.go | 91 +++++++++++++++++++++++++++++++++++ internal/store/store.go | 12 ++++- 5 files changed, 211 insertions(+), 23 deletions(-) diff --git a/internal/queue/queue.go b/internal/queue/queue.go index 389ac98..201ddf2 100644 --- a/internal/queue/queue.go +++ b/internal/queue/queue.go @@ -4,6 +4,7 @@ package queue import ( "context" "errors" + "fmt" "log" "sync" "time" @@ -40,7 +41,8 @@ func Backoff(failures int) time.Duration { type Stats struct { Sent int // delivered to the relay Retried int // failed temporarily, scheduled again - Dropped int // removed: permanent failure or too old + Dropped int // removed: the mailbox does not exist + Failed int // given up for good (refused by the relay, or too old): kept visible for a week Bounced int // dropped deliveries that counted as a hard bounce } @@ -123,6 +125,9 @@ func (w *Worker) RunOnce(ctx context.Context) (Stats, error) { if conc <= 0 { conc = defaultConcurrency } + if _, err := w.Store.PurgeFailedOutbox(); err != nil { + log.Printf("queue: purge old failures: %v", err) + } due, err := w.Store.DueOutbox(batch) if err != nil { return Stats{}, err @@ -148,6 +153,7 @@ func (w *Worker) RunOnce(ctx context.Context) (Stats, error) { stats.Sent += o.Sent stats.Retried += o.Retried stats.Dropped += o.Dropped + stats.Failed += o.Failed stats.Bounced += o.Bounced mu.Unlock() }(it) @@ -190,9 +196,10 @@ func (w *Worker) deliver(ctx context.Context, it store.OutboxItem) Stats { return Stats{Sent: 1} case ctx.Err() != nil: return Stats{} // shutting down: the delivery stays queued, unchanged - case outbound.IsPermanent(err): + case outbound.IsBadMailbox(err): + // The address does not exist: nothing more to say about it, but count it against the member. out := Stats{Dropped: 1} - if outbound.IsBadMailbox(err) && it.MemberID != 0 { + if it.MemberID != 0 { if _, e := w.Store.RecordHardBounce(it.MemberID, store.BounceThreshold); e == nil { out.Bounced = 1 } @@ -200,12 +207,14 @@ func (w *Worker) deliver(ctx context.Context, it store.OutboxItem) Stats { log.Printf("queue: drop %s: %v", it.Rcpt, err) w.complete(it) return out + case outbound.IsPermanent(err): + // The relay refused the message itself (policy, size, line length...): nobody gets it, so + // keep the delivery listed with the reason instead of making it vanish. + return w.giveUp(it, fmt.Sprintf("refused by the relay: %v", err)) } now := w.Store.Now() if now.Sub(it.CreatedAt) > MaxAge { - log.Printf("queue: give up on %s after %s: %v", it.Rcpt, MaxAge, err) - w.complete(it) - return Stats{Dropped: 1} + return w.giveUp(it, fmt.Sprintf("gave up after %s: %v", MaxAge, err)) } if e := w.Store.RetryOutbox(it.ID, now.Add(Backoff(it.Attempts+1)), err.Error()); e != nil { log.Printf("queue: reschedule %d: %v", it.ID, e) @@ -214,6 +223,16 @@ func (w *Worker) deliver(ctx context.Context, it store.OutboxItem) Stats { return Stats{Retried: 1} } +// giveUp marks a delivery as failed for good and logs it. +func (w *Worker) giveUp(it store.OutboxItem, reason string) Stats { + log.Printf("queue: %s: %s", it.Rcpt, reason) + if err := w.Store.FailOutbox(it.ID, reason); err != nil { + log.Printf("queue: mark %d failed: %v", it.ID, err) + return Stats{} + } + return Stats{Failed: 1} +} + func (w *Worker) complete(it store.OutboxItem) { if err := w.Store.CompleteOutbox(it.ID); err != nil && !errors.Is(err, store.ErrNotFound) { log.Printf("queue: remove %d: %v", it.ID, err) diff --git a/internal/queue/queue_test.go b/internal/queue/queue_test.go index f96e6f5..5945d70 100644 --- a/internal/queue/queue_test.go +++ b/internal/queue/queue_test.go @@ -203,15 +203,44 @@ func TestOneFailureDoesNotBlockTheOthers(t *testing.T) { } } -func TestPermanentFailureDropsWithoutBouncing(t *testing.T) { +func TestPermanentFailureIsKeptVisibleAndNeverRetried(t *testing.T) { e := setup(t) e.enqueue(t, 1) + calls := 0 e.snd.fn = func(context.Context, string, []string) error { + calls++ return &smtp.SMTPError{Code: 554, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "relay access denied"} } st, _ := e.w.RunOnce(context.Background()) - if st.Dropped != 1 || st.Bounced != 0 || e.pending(t) != 0 { - t.Fatalf("%+v pending=%d", st, e.pending(t)) + if st.Failed != 1 || st.Dropped != 0 || st.Bounced != 0 { + t.Fatalf("%+v", st) + } + stats, _ := e.s.OutboxStats() + if stats.Pending != 0 || stats.Failed != 1 { + t.Fatalf("the delivery is not lost, it is listed as failed: %+v", stats) + } + list, _ := e.s.ListOutbox(10) + if len(list) != 1 || !list[0].Failed || !strings.Contains(list[0].LastError, "relay access denied") { + t.Fatalf("the admin sees why: %+v", list) + } + e.clk.set(t0.Add(48 * time.Hour)) + e.w.RunOnce(context.Background()) + if calls != 1 { + t.Fatalf("a refused delivery is never retried, relay called %d times", calls) + } +} + +func TestFailedDeliveriesAreForgottenAfterAWeekByTheWorker(t *testing.T) { + e := setup(t) + e.enqueue(t, 1) + e.snd.fn = func(context.Context, string, []string) error { + return &smtp.SMTPError{Code: 554, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "no"} + } + e.w.RunOnce(context.Background()) + e.clk.set(t0.Add(8 * 24 * time.Hour)) + e.w.RunOnce(context.Background()) + if stats, _ := e.s.OutboxStats(); stats.Failed != 0 { + t.Fatalf("an old failure is purged on the next run: %+v", stats) } } @@ -252,7 +281,7 @@ func TestOtherPermanentRejectionsNeverCountAgainstTheMember(t *testing.T) { for i := 0; i < 5; i++ { e.s.Enqueue([]byte("x"), []store.NewOutbound{{From: "b@x.org", Rcpt: "ok@x.com", MemberID: ms[0].ID}}) st, _ := e.w.RunOnce(context.Background()) - if st.Dropped != 1 || st.Bounced != 0 { + if st.Failed != 1 || st.Bounced != 0 { t.Fatalf("round %d: %+v", i, st) } } @@ -288,9 +317,13 @@ func TestGivesUpAfterMaxAge(t *testing.T) { // The retry delay may not have passed yet; move far enough. e.clk.set(e.clk.now().Add(7 * time.Hour)) st, _ := e.w.RunOnce(context.Background()) - if st.Dropped != 1 || e.pending(t) != 0 { + if st.Failed != 1 || e.pending(t) != 0 { t.Fatalf("too old: %+v pending=%d", st, e.pending(t)) } + list, _ := e.s.ListOutbox(10) + if len(list) != 1 || !list[0].Failed || !strings.Contains(list[0].LastError, "gave up") || !strings.Contains(list[0].LastError, "still down") { + t.Fatalf("it stays listed with the reason: %+v", list) + } } func TestOldMailIsStillDeliveredIfTheRelayWorks(t *testing.T) { diff --git a/internal/store/outbox.go b/internal/store/outbox.go index 34f0419..aaa3e20 100644 --- a/internal/store/outbox.go +++ b/internal/store/outbox.go @@ -26,15 +26,22 @@ type OutboxItem struct { NextAt time.Time LastError string CreatedAt time.Time + // Failed deliveries are never retried (the relay refused them for good, or they are too old). + // They stay listed for a week so the admin can see what was lost and why. + Failed bool } // OutboxStats summarises the queue. type OutboxStats struct { - Pending int // every queued delivery - Retrying int // deliveries that already failed at least once - Oldest time.Time // zero when the queue is empty + Pending int // deliveries still waiting to be sent + Retrying int // of those, the ones that already failed at least once + Failed int // given up: kept for a week for the admin to see + Oldest time.Time // oldest waiting delivery; zero when none waits } +// failedTTL is how long a failed delivery stays listed. +const failedTTL = 7 * 24 * time.Hour + // Enqueue stores a message and one delivery per recipient, all or nothing. func (s *Store) Enqueue(msg []byte, items []NewOutbound) error { if len(items) == 0 { @@ -91,13 +98,13 @@ func enqueueTx(tx *sql.Tx, now int64, msg []byte, items []NewOutbound) error { return nil } -const outboxCols = `id, msg_id, envelope_from, rcpt, member_id, prefix, attempts, next_at, last_error, created_at` +const outboxCols = `id, msg_id, envelope_from, rcpt, member_id, prefix, attempts, next_at, last_error, created_at, failed` func scanOutbox(row interface{ Scan(...any) error }) (OutboxItem, error) { var it OutboxItem - var next, created int64 - err := row.Scan(&it.ID, &it.MsgID, &it.From, &it.Rcpt, &it.MemberID, &it.Prefix, &it.Attempts, &next, &it.LastError, &created) - it.NextAt, it.CreatedAt = time.Unix(next, 0), time.Unix(created, 0) + var next, created, failed int64 + err := row.Scan(&it.ID, &it.MsgID, &it.From, &it.Rcpt, &it.MemberID, &it.Prefix, &it.Attempts, &next, &it.LastError, &created, &failed) + it.NextAt, it.CreatedAt, it.Failed = time.Unix(next, 0), time.Unix(created, 0), failed != 0 return it, err } @@ -120,13 +127,13 @@ func (s *Store) queryOutbox(q string, args ...any) ([]OutboxItem, error) { // DueOutbox returns up to limit deliveries whose time has come, longest waiting first. func (s *Store) DueOutbox(limit int) ([]OutboxItem, error) { - return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox WHERE next_at<=? ORDER BY next_at, id LIMIT ?`, + return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox WHERE failed=0 AND next_at<=? ORDER BY next_at, id LIMIT ?`, s.Now().Unix(), limit) } // ListOutbox returns up to limit deliveries for display, failing ones first. func (s *Store) ListOutbox(limit int) ([]OutboxItem, error) { - return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox ORDER BY attempts DESC, next_at, id LIMIT ?`, limit) + return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox ORDER BY failed DESC, attempts DESC, next_at, id LIMIT ?`, limit) } // OutboxData returns the shared message of a queued delivery. @@ -174,10 +181,38 @@ func (s *Store) RetryOutbox(id int64, next time.Time, lastError string) error { func (s *Store) OutboxStats() (OutboxStats, error) { var st OutboxStats var oldest sql.NullInt64 - err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(SUM(attempts>0),0), MIN(created_at) FROM outbox`). - Scan(&st.Pending, &st.Retrying, &oldest) + err := s.db.QueryRow(`SELECT COALESCE(SUM(failed=0),0), COALESCE(SUM(failed=0 AND attempts>0),0), COALESCE(SUM(failed),0), + MIN(CASE WHEN failed=0 THEN created_at END) FROM outbox`).Scan(&st.Pending, &st.Retrying, &st.Failed, &oldest) if oldest.Valid { st.Oldest = time.Unix(oldest.Int64, 0) } return st, err } + +// FailOutbox gives up on a delivery for good, keeping it listed with the reason. Its time is the +// moment of the failure, from which PurgeFailedOutbox counts. +func (s *Store) FailOutbox(id int64, reason string) error { + return affected(s.db.Exec(`UPDATE outbox SET failed=1, attempts=attempts+1, last_error=?, next_at=? WHERE id=?`, + reason, s.Now().Unix(), id)) +} + +// PurgeFailedOutbox forgets failed deliveries older than a week, and the stored messages nobody +// needs any more. It returns how many deliveries were removed. +func (s *Store) PurgeFailedOutbox() (int, error) { + tx, err := s.db.Begin() + if err != nil { + return 0, err + } + defer tx.Rollback() + res, err := tx.Exec(`DELETE FROM outbox WHERE failed=1 AND next_at 0 { + if _, err := tx.Exec(`DELETE FROM outbox_msgs WHERE id NOT IN (SELECT msg_id FROM outbox)`); err != nil { + return 0, err + } + } + return int(n), tx.Commit() +} diff --git a/internal/store/outbox_test.go b/internal/store/outbox_test.go index d30e8cf..bc11393 100644 --- a/internal/store/outbox_test.go +++ b/internal/store/outbox_test.go @@ -259,3 +259,94 @@ func TestOnlyOneOfSeveralSimultaneousApprovalsWins(t *testing.T) { t.Fatalf("the message must be queued exactly once for 3 recipients, pending = %d", st.Pending) } } + +func TestFailedDeliveriesStayVisibleButAreNeverRetried(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + s.Enqueue([]byte("body"), items(3)) + due, _ := s.DueOutbox(10) + if err := s.FailOutbox(due[0].ID, "relay said: 554 policy rejection"); err != nil { + t.Fatal(err) + } + if err := s.FailOutbox(999, "x"); !errors.Is(err, ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } + + got, _ := s.DueOutbox(10) + if len(got) != 2 { + t.Fatalf("a failed delivery is not due again: %d due", len(got)) + } + now = now.Add(24 * time.Hour) + if got, _ := s.DueOutbox(10); len(got) != 2 { + t.Fatal("not even a day later") + } + + st, _ := s.OutboxStats() + if st.Pending != 2 || st.Failed != 1 || st.Retrying != 0 { + t.Fatalf("stats: %+v", st) + } + list, _ := s.ListOutbox(10) + if len(list) != 3 || !list[0].Failed || list[0].ID != due[0].ID || list[0].LastError != "relay said: 554 policy rejection" { + t.Fatalf("failed deliveries come first, with the reason: %+v", list) + } + // The admin can discard it, like any other. + if err := s.CompleteOutbox(due[0].ID); err != nil { + t.Fatal(err) + } + if st, _ := s.OutboxStats(); st.Failed != 0 { + t.Fatalf("%+v", st) + } +} + +func TestFailedDeliveriesAreForgottenAfterAWeek(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + s.Enqueue([]byte("old body"), items(1)) + s.Enqueue([]byte("new body"), items(1)) + due, _ := s.DueOutbox(10) + s.FailOutbox(due[0].ID, "old failure") + now = now.Add(5 * 24 * time.Hour) + s.FailOutbox(due[1].ID, "recent failure") + + now = now.Add(3 * 24 * time.Hour) // the first failure is now 8 days old, the second 3 + n, err := s.PurgeFailedOutbox() + if err != nil || n != 1 { + t.Fatalf("purged %d: %v", n, err) + } + list, _ := s.ListOutbox(10) + if len(list) != 1 || list[0].LastError != "recent failure" { + t.Fatalf("list: %+v", list) + } + if _, err := s.OutboxData(due[0].MsgID); !errors.Is(err, ErrNotFound) { + t.Fatal("the stored message of a purged delivery goes too") + } + if _, err := s.OutboxData(due[1].MsgID); err != nil { + t.Fatalf("the recent one keeps its message: %v", err) + } + // Waiting and retrying deliveries are never purged, however old. + s.Enqueue([]byte("waiting"), items(1)) + now = now.Add(365 * 24 * time.Hour) + s.PurgeFailedOutbox() + if st, _ := s.OutboxStats(); st.Pending != 1 { + t.Fatalf("a waiting delivery must survive: %+v", st) + } +} + +func TestMigrationAddsTheFailedColumn(t *testing.T) { + path := t.TempDir() + "/old.db" + s, _ := Open(path) + s.Enqueue([]byte("x"), items(1)) + s.db.Exec(`ALTER TABLE outbox DROP COLUMN failed`) + s.Close() + s2, err := Open(path) + if err != nil { + t.Fatalf("migrate: %v", err) + } + defer s2.Close() + due, _ := s2.DueOutbox(10) + if len(due) != 1 || due[0].Failed { + t.Fatalf("an existing delivery stays waiting: %+v", due) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index e1e205b..cb27ebb 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -184,7 +184,8 @@ CREATE TABLE IF NOT EXISTS outbox ( attempts INTEGER NOT NULL DEFAULT 0, next_at INTEGER NOT NULL, last_error TEXT NOT NULL DEFAULT '', - created_at INTEGER NOT NULL + created_at INTEGER NOT NULL, + failed INTEGER NOT NULL DEFAULT 0 ); CREATE INDEX IF NOT EXISTS outbox_due ON outbox(next_at, id); CREATE TABLE IF NOT EXISTS settings ( @@ -257,6 +258,15 @@ func migrate(db *sql.DB) error { } } } + if ok, err := hasColumn(db, "outbox", "msg_id"); err != nil { + return err + } else if ok { + if has, _ := hasColumn(db, "outbox", "failed"); !has { + if _, err := db.Exec(`ALTER TABLE outbox ADD COLUMN failed INTEGER NOT NULL DEFAULT 0`); err != nil { + return err + } + } + } if ok, err := hasColumn(db, "held", "list_id"); err != nil { return err } else if ok {