fix: keep refused deliveries visible instead of dropping them silently
- Add failed deliveries: a permanent refusal by the relay (policy, size, line length) or the 3-day give-up marks the delivery failed, with the reason, and never retries it - Keep failed deliveries listed for a week, then purge them and their stored message - A non-existent mailbox still drops the delivery and counts a hard bounce - Migrate the outbox table
This commit is contained in:
parent
e4e08cd25e
commit
bb0858ef87
5 changed files with 211 additions and 23 deletions
|
|
@ -4,6 +4,7 @@ package queue
|
|||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
|
@ -40,7 +41,8 @@ func Backoff(failures int) time.Duration {
|
|||
type Stats struct {
|
||||
Sent int // delivered to the relay
|
||||
Retried int // failed temporarily, scheduled again
|
||||
Dropped int // removed: permanent failure or too old
|
||||
Dropped int // removed: the mailbox does not exist
|
||||
Failed int // given up for good (refused by the relay, or too old): kept visible for a week
|
||||
Bounced int // dropped deliveries that counted as a hard bounce
|
||||
}
|
||||
|
||||
|
|
@ -123,6 +125,9 @@ func (w *Worker) RunOnce(ctx context.Context) (Stats, error) {
|
|||
if conc <= 0 {
|
||||
conc = defaultConcurrency
|
||||
}
|
||||
if _, err := w.Store.PurgeFailedOutbox(); err != nil {
|
||||
log.Printf("queue: purge old failures: %v", err)
|
||||
}
|
||||
due, err := w.Store.DueOutbox(batch)
|
||||
if err != nil {
|
||||
return Stats{}, err
|
||||
|
|
@ -148,6 +153,7 @@ func (w *Worker) RunOnce(ctx context.Context) (Stats, error) {
|
|||
stats.Sent += o.Sent
|
||||
stats.Retried += o.Retried
|
||||
stats.Dropped += o.Dropped
|
||||
stats.Failed += o.Failed
|
||||
stats.Bounced += o.Bounced
|
||||
mu.Unlock()
|
||||
}(it)
|
||||
|
|
@ -190,9 +196,10 @@ func (w *Worker) deliver(ctx context.Context, it store.OutboxItem) Stats {
|
|||
return Stats{Sent: 1}
|
||||
case ctx.Err() != nil:
|
||||
return Stats{} // shutting down: the delivery stays queued, unchanged
|
||||
case outbound.IsPermanent(err):
|
||||
case outbound.IsBadMailbox(err):
|
||||
// The address does not exist: nothing more to say about it, but count it against the member.
|
||||
out := Stats{Dropped: 1}
|
||||
if outbound.IsBadMailbox(err) && it.MemberID != 0 {
|
||||
if it.MemberID != 0 {
|
||||
if _, e := w.Store.RecordHardBounce(it.MemberID, store.BounceThreshold); e == nil {
|
||||
out.Bounced = 1
|
||||
}
|
||||
|
|
@ -200,12 +207,14 @@ func (w *Worker) deliver(ctx context.Context, it store.OutboxItem) Stats {
|
|||
log.Printf("queue: drop %s: %v", it.Rcpt, err)
|
||||
w.complete(it)
|
||||
return out
|
||||
case outbound.IsPermanent(err):
|
||||
// The relay refused the message itself (policy, size, line length...): nobody gets it, so
|
||||
// keep the delivery listed with the reason instead of making it vanish.
|
||||
return w.giveUp(it, fmt.Sprintf("refused by the relay: %v", err))
|
||||
}
|
||||
now := w.Store.Now()
|
||||
if now.Sub(it.CreatedAt) > MaxAge {
|
||||
log.Printf("queue: give up on %s after %s: %v", it.Rcpt, MaxAge, err)
|
||||
w.complete(it)
|
||||
return Stats{Dropped: 1}
|
||||
return w.giveUp(it, fmt.Sprintf("gave up after %s: %v", MaxAge, err))
|
||||
}
|
||||
if e := w.Store.RetryOutbox(it.ID, now.Add(Backoff(it.Attempts+1)), err.Error()); e != nil {
|
||||
log.Printf("queue: reschedule %d: %v", it.ID, e)
|
||||
|
|
@ -214,6 +223,16 @@ func (w *Worker) deliver(ctx context.Context, it store.OutboxItem) Stats {
|
|||
return Stats{Retried: 1}
|
||||
}
|
||||
|
||||
// giveUp marks a delivery as failed for good and logs it.
|
||||
func (w *Worker) giveUp(it store.OutboxItem, reason string) Stats {
|
||||
log.Printf("queue: %s: %s", it.Rcpt, reason)
|
||||
if err := w.Store.FailOutbox(it.ID, reason); err != nil {
|
||||
log.Printf("queue: mark %d failed: %v", it.ID, err)
|
||||
return Stats{}
|
||||
}
|
||||
return Stats{Failed: 1}
|
||||
}
|
||||
|
||||
func (w *Worker) complete(it store.OutboxItem) {
|
||||
if err := w.Store.CompleteOutbox(it.ID); err != nil && !errors.Is(err, store.ErrNotFound) {
|
||||
log.Printf("queue: remove %d: %v", it.ID, err)
|
||||
|
|
|
|||
|
|
@ -203,15 +203,44 @@ func TestOneFailureDoesNotBlockTheOthers(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestPermanentFailureDropsWithoutBouncing(t *testing.T) {
|
||||
func TestPermanentFailureIsKeptVisibleAndNeverRetried(t *testing.T) {
|
||||
e := setup(t)
|
||||
e.enqueue(t, 1)
|
||||
calls := 0
|
||||
e.snd.fn = func(context.Context, string, []string) error {
|
||||
calls++
|
||||
return &smtp.SMTPError{Code: 554, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "relay access denied"}
|
||||
}
|
||||
st, _ := e.w.RunOnce(context.Background())
|
||||
if st.Dropped != 1 || st.Bounced != 0 || e.pending(t) != 0 {
|
||||
t.Fatalf("%+v pending=%d", st, e.pending(t))
|
||||
if st.Failed != 1 || st.Dropped != 0 || st.Bounced != 0 {
|
||||
t.Fatalf("%+v", st)
|
||||
}
|
||||
stats, _ := e.s.OutboxStats()
|
||||
if stats.Pending != 0 || stats.Failed != 1 {
|
||||
t.Fatalf("the delivery is not lost, it is listed as failed: %+v", stats)
|
||||
}
|
||||
list, _ := e.s.ListOutbox(10)
|
||||
if len(list) != 1 || !list[0].Failed || !strings.Contains(list[0].LastError, "relay access denied") {
|
||||
t.Fatalf("the admin sees why: %+v", list)
|
||||
}
|
||||
e.clk.set(t0.Add(48 * time.Hour))
|
||||
e.w.RunOnce(context.Background())
|
||||
if calls != 1 {
|
||||
t.Fatalf("a refused delivery is never retried, relay called %d times", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailedDeliveriesAreForgottenAfterAWeekByTheWorker(t *testing.T) {
|
||||
e := setup(t)
|
||||
e.enqueue(t, 1)
|
||||
e.snd.fn = func(context.Context, string, []string) error {
|
||||
return &smtp.SMTPError{Code: 554, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "no"}
|
||||
}
|
||||
e.w.RunOnce(context.Background())
|
||||
e.clk.set(t0.Add(8 * 24 * time.Hour))
|
||||
e.w.RunOnce(context.Background())
|
||||
if stats, _ := e.s.OutboxStats(); stats.Failed != 0 {
|
||||
t.Fatalf("an old failure is purged on the next run: %+v", stats)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -252,7 +281,7 @@ func TestOtherPermanentRejectionsNeverCountAgainstTheMember(t *testing.T) {
|
|||
for i := 0; i < 5; i++ {
|
||||
e.s.Enqueue([]byte("x"), []store.NewOutbound{{From: "b@x.org", Rcpt: "ok@x.com", MemberID: ms[0].ID}})
|
||||
st, _ := e.w.RunOnce(context.Background())
|
||||
if st.Dropped != 1 || st.Bounced != 0 {
|
||||
if st.Failed != 1 || st.Bounced != 0 {
|
||||
t.Fatalf("round %d: %+v", i, st)
|
||||
}
|
||||
}
|
||||
|
|
@ -288,9 +317,13 @@ func TestGivesUpAfterMaxAge(t *testing.T) {
|
|||
// The retry delay may not have passed yet; move far enough.
|
||||
e.clk.set(e.clk.now().Add(7 * time.Hour))
|
||||
st, _ := e.w.RunOnce(context.Background())
|
||||
if st.Dropped != 1 || e.pending(t) != 0 {
|
||||
if st.Failed != 1 || e.pending(t) != 0 {
|
||||
t.Fatalf("too old: %+v pending=%d", st, e.pending(t))
|
||||
}
|
||||
list, _ := e.s.ListOutbox(10)
|
||||
if len(list) != 1 || !list[0].Failed || !strings.Contains(list[0].LastError, "gave up") || !strings.Contains(list[0].LastError, "still down") {
|
||||
t.Fatalf("it stays listed with the reason: %+v", list)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOldMailIsStillDeliveredIfTheRelayWorks(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -26,15 +26,22 @@ type OutboxItem struct {
|
|||
NextAt time.Time
|
||||
LastError string
|
||||
CreatedAt time.Time
|
||||
// Failed deliveries are never retried (the relay refused them for good, or they are too old).
|
||||
// They stay listed for a week so the admin can see what was lost and why.
|
||||
Failed bool
|
||||
}
|
||||
|
||||
// OutboxStats summarises the queue.
|
||||
type OutboxStats struct {
|
||||
Pending int // every queued delivery
|
||||
Retrying int // deliveries that already failed at least once
|
||||
Oldest time.Time // zero when the queue is empty
|
||||
Pending int // deliveries still waiting to be sent
|
||||
Retrying int // of those, the ones that already failed at least once
|
||||
Failed int // given up: kept for a week for the admin to see
|
||||
Oldest time.Time // oldest waiting delivery; zero when none waits
|
||||
}
|
||||
|
||||
// failedTTL is how long a failed delivery stays listed.
|
||||
const failedTTL = 7 * 24 * time.Hour
|
||||
|
||||
// Enqueue stores a message and one delivery per recipient, all or nothing.
|
||||
func (s *Store) Enqueue(msg []byte, items []NewOutbound) error {
|
||||
if len(items) == 0 {
|
||||
|
|
@ -91,13 +98,13 @@ func enqueueTx(tx *sql.Tx, now int64, msg []byte, items []NewOutbound) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
const outboxCols = `id, msg_id, envelope_from, rcpt, member_id, prefix, attempts, next_at, last_error, created_at`
|
||||
const outboxCols = `id, msg_id, envelope_from, rcpt, member_id, prefix, attempts, next_at, last_error, created_at, failed`
|
||||
|
||||
func scanOutbox(row interface{ Scan(...any) error }) (OutboxItem, error) {
|
||||
var it OutboxItem
|
||||
var next, created int64
|
||||
err := row.Scan(&it.ID, &it.MsgID, &it.From, &it.Rcpt, &it.MemberID, &it.Prefix, &it.Attempts, &next, &it.LastError, &created)
|
||||
it.NextAt, it.CreatedAt = time.Unix(next, 0), time.Unix(created, 0)
|
||||
var next, created, failed int64
|
||||
err := row.Scan(&it.ID, &it.MsgID, &it.From, &it.Rcpt, &it.MemberID, &it.Prefix, &it.Attempts, &next, &it.LastError, &created, &failed)
|
||||
it.NextAt, it.CreatedAt, it.Failed = time.Unix(next, 0), time.Unix(created, 0), failed != 0
|
||||
return it, err
|
||||
}
|
||||
|
||||
|
|
@ -120,13 +127,13 @@ func (s *Store) queryOutbox(q string, args ...any) ([]OutboxItem, error) {
|
|||
|
||||
// DueOutbox returns up to limit deliveries whose time has come, longest waiting first.
|
||||
func (s *Store) DueOutbox(limit int) ([]OutboxItem, error) {
|
||||
return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox WHERE next_at<=? ORDER BY next_at, id LIMIT ?`,
|
||||
return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox WHERE failed=0 AND next_at<=? ORDER BY next_at, id LIMIT ?`,
|
||||
s.Now().Unix(), limit)
|
||||
}
|
||||
|
||||
// ListOutbox returns up to limit deliveries for display, failing ones first.
|
||||
func (s *Store) ListOutbox(limit int) ([]OutboxItem, error) {
|
||||
return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox ORDER BY attempts DESC, next_at, id LIMIT ?`, limit)
|
||||
return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox ORDER BY failed DESC, attempts DESC, next_at, id LIMIT ?`, limit)
|
||||
}
|
||||
|
||||
// OutboxData returns the shared message of a queued delivery.
|
||||
|
|
@ -174,10 +181,38 @@ func (s *Store) RetryOutbox(id int64, next time.Time, lastError string) error {
|
|||
func (s *Store) OutboxStats() (OutboxStats, error) {
|
||||
var st OutboxStats
|
||||
var oldest sql.NullInt64
|
||||
err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(SUM(attempts>0),0), MIN(created_at) FROM outbox`).
|
||||
Scan(&st.Pending, &st.Retrying, &oldest)
|
||||
err := s.db.QueryRow(`SELECT COALESCE(SUM(failed=0),0), COALESCE(SUM(failed=0 AND attempts>0),0), COALESCE(SUM(failed),0),
|
||||
MIN(CASE WHEN failed=0 THEN created_at END) FROM outbox`).Scan(&st.Pending, &st.Retrying, &st.Failed, &oldest)
|
||||
if oldest.Valid {
|
||||
st.Oldest = time.Unix(oldest.Int64, 0)
|
||||
}
|
||||
return st, err
|
||||
}
|
||||
|
||||
// FailOutbox gives up on a delivery for good, keeping it listed with the reason. Its time is the
|
||||
// moment of the failure, from which PurgeFailedOutbox counts.
|
||||
func (s *Store) FailOutbox(id int64, reason string) error {
|
||||
return affected(s.db.Exec(`UPDATE outbox SET failed=1, attempts=attempts+1, last_error=?, next_at=? WHERE id=?`,
|
||||
reason, s.Now().Unix(), id))
|
||||
}
|
||||
|
||||
// PurgeFailedOutbox forgets failed deliveries older than a week, and the stored messages nobody
|
||||
// needs any more. It returns how many deliveries were removed.
|
||||
func (s *Store) PurgeFailedOutbox() (int, error) {
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
res, err := tx.Exec(`DELETE FROM outbox WHERE failed=1 AND next_at<?`, s.Now().Add(-failedTTL).Unix())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n, _ := res.RowsAffected()
|
||||
if n > 0 {
|
||||
if _, err := tx.Exec(`DELETE FROM outbox_msgs WHERE id NOT IN (SELECT msg_id FROM outbox)`); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return int(n), tx.Commit()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -259,3 +259,94 @@ func TestOnlyOneOfSeveralSimultaneousApprovalsWins(t *testing.T) {
|
|||
t.Fatalf("the message must be queued exactly once for 3 recipients, pending = %d", st.Pending)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailedDeliveriesStayVisibleButAreNeverRetried(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
now := time.Unix(1_000_000, 0)
|
||||
s.Now = func() time.Time { return now }
|
||||
s.Enqueue([]byte("body"), items(3))
|
||||
due, _ := s.DueOutbox(10)
|
||||
if err := s.FailOutbox(due[0].ID, "relay said: 554 policy rejection"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.FailOutbox(999, "x"); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("want ErrNotFound, got %v", err)
|
||||
}
|
||||
|
||||
got, _ := s.DueOutbox(10)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("a failed delivery is not due again: %d due", len(got))
|
||||
}
|
||||
now = now.Add(24 * time.Hour)
|
||||
if got, _ := s.DueOutbox(10); len(got) != 2 {
|
||||
t.Fatal("not even a day later")
|
||||
}
|
||||
|
||||
st, _ := s.OutboxStats()
|
||||
if st.Pending != 2 || st.Failed != 1 || st.Retrying != 0 {
|
||||
t.Fatalf("stats: %+v", st)
|
||||
}
|
||||
list, _ := s.ListOutbox(10)
|
||||
if len(list) != 3 || !list[0].Failed || list[0].ID != due[0].ID || list[0].LastError != "relay said: 554 policy rejection" {
|
||||
t.Fatalf("failed deliveries come first, with the reason: %+v", list)
|
||||
}
|
||||
// The admin can discard it, like any other.
|
||||
if err := s.CompleteOutbox(due[0].ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st, _ := s.OutboxStats(); st.Failed != 0 {
|
||||
t.Fatalf("%+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailedDeliveriesAreForgottenAfterAWeek(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
now := time.Unix(1_000_000, 0)
|
||||
s.Now = func() time.Time { return now }
|
||||
s.Enqueue([]byte("old body"), items(1))
|
||||
s.Enqueue([]byte("new body"), items(1))
|
||||
due, _ := s.DueOutbox(10)
|
||||
s.FailOutbox(due[0].ID, "old failure")
|
||||
now = now.Add(5 * 24 * time.Hour)
|
||||
s.FailOutbox(due[1].ID, "recent failure")
|
||||
|
||||
now = now.Add(3 * 24 * time.Hour) // the first failure is now 8 days old, the second 3
|
||||
n, err := s.PurgeFailedOutbox()
|
||||
if err != nil || n != 1 {
|
||||
t.Fatalf("purged %d: %v", n, err)
|
||||
}
|
||||
list, _ := s.ListOutbox(10)
|
||||
if len(list) != 1 || list[0].LastError != "recent failure" {
|
||||
t.Fatalf("list: %+v", list)
|
||||
}
|
||||
if _, err := s.OutboxData(due[0].MsgID); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatal("the stored message of a purged delivery goes too")
|
||||
}
|
||||
if _, err := s.OutboxData(due[1].MsgID); err != nil {
|
||||
t.Fatalf("the recent one keeps its message: %v", err)
|
||||
}
|
||||
// Waiting and retrying deliveries are never purged, however old.
|
||||
s.Enqueue([]byte("waiting"), items(1))
|
||||
now = now.Add(365 * 24 * time.Hour)
|
||||
s.PurgeFailedOutbox()
|
||||
if st, _ := s.OutboxStats(); st.Pending != 1 {
|
||||
t.Fatalf("a waiting delivery must survive: %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrationAddsTheFailedColumn(t *testing.T) {
|
||||
path := t.TempDir() + "/old.db"
|
||||
s, _ := Open(path)
|
||||
s.Enqueue([]byte("x"), items(1))
|
||||
s.db.Exec(`ALTER TABLE outbox DROP COLUMN failed`)
|
||||
s.Close()
|
||||
s2, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
defer s2.Close()
|
||||
due, _ := s2.DueOutbox(10)
|
||||
if len(due) != 1 || due[0].Failed {
|
||||
t.Fatalf("an existing delivery stays waiting: %+v", due)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -184,7 +184,8 @@ CREATE TABLE IF NOT EXISTS outbox (
|
|||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
next_at INTEGER NOT NULL,
|
||||
last_error TEXT NOT NULL DEFAULT '',
|
||||
created_at INTEGER NOT NULL
|
||||
created_at INTEGER NOT NULL,
|
||||
failed INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS outbox_due ON outbox(next_at, id);
|
||||
CREATE TABLE IF NOT EXISTS settings (
|
||||
|
|
@ -257,6 +258,15 @@ func migrate(db *sql.DB) error {
|
|||
}
|
||||
}
|
||||
}
|
||||
if ok, err := hasColumn(db, "outbox", "msg_id"); err != nil {
|
||||
return err
|
||||
} else if ok {
|
||||
if has, _ := hasColumn(db, "outbox", "failed"); !has {
|
||||
if _, err := db.Exec(`ALTER TABLE outbox ADD COLUMN failed INTEGER NOT NULL DEFAULT 0`); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if ok, err := hasColumn(db, "held", "list_id"); err != nil {
|
||||
return err
|
||||
} else if ok {
|
||||
|
|
|
|||
Loading…
Reference in a new issue