package queue import ( "context" "errors" "strings" "sync" "sync/atomic" "testing" "time" "github.com/emersion/go-smtp" "codeberg.org/squidd/parsimony/internal/outbound" "codeberg.org/squidd/parsimony/internal/store" ) var t0 = time.Unix(1_900_000_000, 0) // script is a Sender whose behaviour the test controls. type script struct { mu sync.Mutex sent []outbound.Sent fn func(ctx context.Context, from string, to []string) error // nil: succeed calls int32 } func (s *script) Send(ctx context.Context, from string, to []string, data []byte) error { atomic.AddInt32(&s.calls, 1) if s.fn != nil { if err := s.fn(ctx, from, to); err != nil { return err } } s.mu.Lock() defer s.mu.Unlock() s.sent = append(s.sent, outbound.Sent{From: from, To: append([]string(nil), to...), Data: append([]byte(nil), data...)}) return nil } func (s *script) count() int { s.mu.Lock() defer s.mu.Unlock() return len(s.sent) } // clock is a test clock that the worker goroutine may read while the test moves it. type clock struct { mu sync.Mutex t time.Time } func (c *clock) now() time.Time { c.mu.Lock() defer c.mu.Unlock() return c.t } func (c *clock) set(t time.Time) { c.mu.Lock() c.t = t c.mu.Unlock() } type env struct { s *store.Store w *Worker snd *script clk *clock } func setup(t *testing.T) *env { t.Helper() s, err := store.Open(":memory:") if err != nil { t.Fatal(err) } t.Cleanup(func() { s.Close() }) clk := &clock{t: t0} s.Now = clk.now snd := &script{} return &env{s: s, snd: snd, clk: clk, w: &Worker{Store: s, Sender: snd}} } func (e *env) enqueue(t *testing.T, n int) { t.Helper() var items []store.NewOutbound for i := 0; i < n; i++ { items = append(items, store.NewOutbound{From: "bounce@x.org", Rcpt: string(rune('a'+i)) + "@x.com", Prefix: []byte("X-Rcpt: " + string(rune('a'+i)) + "\r\n")}) } if err := e.s.Enqueue([]byte("Subject: hi\r\n\r\nbody\r\n"), items); err != nil { t.Fatal(err) } } func (e *env) pending(t *testing.T) int { t.Helper() st, _ := e.s.OutboxStats() return st.Pending } func TestDeliversPrefixPlusMessage(t *testing.T) { e := setup(t) e.enqueue(t, 2) st, err := e.w.RunOnce(context.Background()) if err != nil || st.Sent != 2 { t.Fatalf("run: %+v %v", st, err) } if e.pending(t) != 0 { t.Fatal("sent items must leave the queue") } got := e.snd.sent[0] if got.From != "bounce@x.org" || len(got.To) != 1 || got.To[0] != "a@x.com" { t.Fatalf("envelope: %+v", got) } if string(got.Data) != "X-Rcpt: a\r\nSubject: hi\r\n\r\nbody\r\n" { t.Fatalf("data = %q", got.Data) } if string(e.snd.sent[1].Data) != "X-Rcpt: b\r\nSubject: hi\r\n\r\nbody\r\n" { t.Fatalf("each recipient has its own prefix: %q", e.snd.sent[1].Data) } } func TestEmptyQueueIsFine(t *testing.T) { e := setup(t) if st, err := e.w.RunOnce(context.Background()); err != nil || st != (Stats{}) { t.Fatalf("%+v %v", st, err) } } func TestBackoffSchedule(t *testing.T) { want := []time.Duration{time.Minute, 5 * time.Minute, 15 * time.Minute, time.Hour, 3 * time.Hour, 6 * time.Hour, 6 * time.Hour, 6 * time.Hour} for i, w := range want { if got := Backoff(i + 1); got != w { t.Errorf("Backoff(%d) = %v, want %v", i+1, got, w) } } if Backoff(0) != time.Minute || Backoff(-3) != time.Minute { t.Error("nonsense input falls back to the first delay") } } func TestTemporaryFailureIsRetriedLater(t *testing.T) { e := setup(t) e.enqueue(t, 1) e.snd.fn = func(context.Context, string, []string) error { return errors.New("connection refused") } st, _ := e.w.RunOnce(context.Background()) if st.Retried != 1 || st.Sent != 0 { t.Fatalf("stats: %+v", st) } list, _ := e.s.ListOutbox(10) if len(list) != 1 || list[0].Attempts != 1 || !strings.Contains(list[0].LastError, "connection refused") || !list[0].NextAt.Equal(t0.Add(time.Minute)) { t.Fatalf("item: %+v", list) } // Not due yet: nothing is attempted. before := atomic.LoadInt32(&e.snd.calls) e.clk.set(t0.Add(59 * time.Second)) e.w.RunOnce(context.Background()) if atomic.LoadInt32(&e.snd.calls) != before { t.Fatal("must wait for the backoff") } // Second failure waits longer. e.clk.set(t0.Add(time.Minute)) e.w.RunOnce(context.Background()) list, _ = e.s.ListOutbox(10) if list[0].Attempts != 2 || !list[0].NextAt.Equal(t0.Add(time.Minute+5*time.Minute)) { t.Fatalf("second failure: %+v", list[0]) } // The relay comes back. e.snd.fn = nil e.clk.set(t0.Add(10 * time.Minute)) st, _ = e.w.RunOnce(context.Background()) if st.Sent != 1 || e.pending(t) != 0 || e.snd.count() != 1 { t.Fatalf("recovery: %+v pending=%d sent=%d", st, e.pending(t), e.snd.count()) } } func TestOneFailureDoesNotBlockTheOthers(t *testing.T) { e := setup(t) e.enqueue(t, 3) e.snd.fn = func(_ context.Context, _ string, to []string) error { if to[0] == "b@x.com" { return errors.New("greylisted") } return nil } st, _ := e.w.RunOnce(context.Background()) if st.Sent != 2 || st.Retried != 1 || e.pending(t) != 1 { t.Fatalf("%+v pending=%d", st, e.pending(t)) } list, _ := e.s.ListOutbox(10) if list[0].Rcpt != "b@x.com" { t.Fatalf("the failed recipient stays queued, got %q", list[0].Rcpt) } } func TestPermanentFailureDropsWithoutBouncing(t *testing.T) { e := setup(t) e.enqueue(t, 1) e.snd.fn = func(context.Context, string, []string) error { return &smtp.SMTPError{Code: 554, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "relay access denied"} } st, _ := e.w.RunOnce(context.Background()) if st.Dropped != 1 || st.Bounced != 0 || e.pending(t) != 0 { t.Fatalf("%+v pending=%d", st, e.pending(t)) } } func TestBadMailboxCountsAHardBounceAndDisablesAfterThree(t *testing.T) { e := setup(t) l, _ := e.s.CreateList("dev", "example.org", "Dev") e.s.AddMember(l.ID, "gone@x.com", "") ms, _ := e.s.Members(l.ID) id := ms[0].ID e.snd.fn = func(context.Context, string, []string) error { return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"} } for i := 1; i <= 3; i++ { e.s.Enqueue([]byte("Subject: x\r\n\r\nx"), []store.NewOutbound{{From: "b@x.org", Rcpt: "gone@x.com", MemberID: id}}) st, _ := e.w.RunOnce(context.Background()) if st.Dropped != 1 || st.Bounced != 1 { t.Fatalf("round %d: %+v", i, st) } m, _ := e.s.MemberByID(id) if m.BounceCount != i || m.Disabled != (i == 3) { t.Fatalf("round %d: member %+v", i, m) } } if e.pending(t) != 0 { t.Fatal("permanent failures leave the queue") } } func TestBadMailboxForSystemMailOrUnknownMemberIsHarmless(t *testing.T) { e := setup(t) e.snd.fn = func(context.Context, string, []string) error { return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"} } e.s.Enqueue([]byte("x"), []store.NewOutbound{ {From: "b@x.org", Rcpt: "a@x.com", MemberID: 0}, // confirmation mail {From: "b@x.org", Rcpt: "b@x.com", MemberID: 404}, // member was deleted meanwhile }) st, err := e.w.RunOnce(context.Background()) if err != nil || st.Dropped != 2 || e.pending(t) != 0 { t.Fatalf("%+v %v pending=%d", st, err, e.pending(t)) } } func TestGivesUpAfterMaxAge(t *testing.T) { e := setup(t) e.enqueue(t, 1) e.snd.fn = func(context.Context, string, []string) error { return errors.New("still down") } e.clk.set(t0.Add(MaxAge - time.Minute)) if st, _ := e.w.RunOnce(context.Background()); st.Retried != 1 { t.Fatalf("just inside the limit is retried: %+v", st) } e.clk.set(t0.Add(MaxAge + time.Minute)) // The retry delay may not have passed yet; move far enough. e.clk.set(e.clk.now().Add(7 * time.Hour)) st, _ := e.w.RunOnce(context.Background()) if st.Dropped != 1 || e.pending(t) != 0 { t.Fatalf("too old: %+v pending=%d", st, e.pending(t)) } } func TestOldMailIsStillDeliveredIfTheRelayWorks(t *testing.T) { e := setup(t) e.enqueue(t, 1) e.clk.set(t0.Add(MaxAge * 2)) if st, _ := e.w.RunOnce(context.Background()); st.Sent != 1 { t.Fatalf("age only matters for failures: %+v", st) } } func TestConcurrencyIsBounded(t *testing.T) { e := setup(t) e.enqueue(t, 10) e.w.Concurrency = 3 var cur, max int32 e.snd.fn = func(ctx context.Context, _ string, _ []string) error { n := atomic.AddInt32(&cur, 1) for { m := atomic.LoadInt32(&max) if n <= m || atomic.CompareAndSwapInt32(&max, m, n) { break } } time.Sleep(20 * time.Millisecond) atomic.AddInt32(&cur, -1) return nil } st, _ := e.w.RunOnce(context.Background()) if st.Sent != 10 { t.Fatalf("%+v", st) } if max != 3 { t.Fatalf("max concurrent sends = %d, want 3", max) } } func TestBatchLimitsOneRun(t *testing.T) { e := setup(t) e.enqueue(t, 5) e.w.Batch = 2 st, _ := e.w.RunOnce(context.Background()) if st.Sent != 2 || e.pending(t) != 3 { t.Fatalf("%+v pending=%d", st, e.pending(t)) } } func TestInterruptedSendStaysQueuedUntouched(t *testing.T) { e := setup(t) e.enqueue(t, 1) ctx, cancel := context.WithCancel(context.Background()) e.snd.fn = func(ctx context.Context, _ string, _ []string) error { cancel() // shutdown arrives while the send is in flight <-ctx.Done() return ctx.Err() } e.w.RunOnce(ctx) list, _ := e.s.ListOutbox(10) if len(list) != 1 || list[0].Attempts != 0 || list[0].LastError != "" { t.Fatalf("a shutdown is not a delivery failure: %+v", list) } } func TestSlowRelayTimesOutAndIsRetried(t *testing.T) { e := setup(t) e.enqueue(t, 1) e.w.Timeout = 30 * time.Millisecond e.snd.fn = func(ctx context.Context, _ string, _ []string) error { <-ctx.Done() return ctx.Err() } st, _ := e.w.RunOnce(context.Background()) if st.Retried != 1 { t.Fatalf("a hung relay is a temporary failure: %+v", st) } list, _ := e.s.ListOutbox(10) if !strings.Contains(list[0].LastError, "deadline") { t.Fatalf("last error: %q", list[0].LastError) } } func waitFor(t *testing.T, what string, cond func() bool) { t.Helper() deadline := time.Now().Add(3 * time.Second) for !cond() { if time.Now().After(deadline) { t.Fatalf("timed out waiting for %s", what) } time.Sleep(5 * time.Millisecond) } } func TestRunDrainsTheQueueInBatchesAndStopsOnCancel(t *testing.T) { e := setup(t) e.enqueue(t, 7) e.w.Batch = 2 e.w.Interval = time.Hour ctx, cancel := context.WithCancel(context.Background()) done := make(chan struct{}) go func() { e.w.Run(ctx); close(done) }() waitFor(t, "7 deliveries", func() bool { return e.snd.count() == 7 }) cancel() select { case <-done: case <-time.After(2 * time.Second): t.Fatal("Run did not stop") } } func TestWakeTriggersAnImmediateRun(t *testing.T) { e := setup(t) e.w.Interval = time.Hour ctx, cancel := context.WithCancel(context.Background()) defer cancel() go e.w.Run(ctx) time.Sleep(50 * time.Millisecond) // the first run finds nothing and goes idle e.enqueue(t, 1) e.w.Wake() waitFor(t, "woken delivery", func() bool { return e.snd.count() == 1 }) } func TestWakeBeforeRunAndRepeatedWakesDoNotBlock(t *testing.T) { e := setup(t) done := make(chan struct{}) go func() { for i := 0; i < 100; i++ { e.w.Wake() } close(done) }() select { case <-done: case <-time.After(time.Second): t.Fatal("Wake blocked") } } func TestIdleRunPollsForDueRetries(t *testing.T) { e := setup(t) e.enqueue(t, 1) fail := int32(1) e.snd.fn = func(context.Context, string, []string) error { if atomic.LoadInt32(&fail) == 1 { return errors.New("down") } return nil } e.w.Interval = 20 * time.Millisecond ctx, cancel := context.WithCancel(context.Background()) defer cancel() go e.w.Run(ctx) waitFor(t, "first failed attempt", func() bool { st, _ := e.s.OutboxStats(); return st.Retrying == 1 }) atomic.StoreInt32(&fail, 0) e.clk.set(t0.Add(2 * time.Minute)) // the backoff has passed; no Wake needed waitFor(t, "retry by polling", func() bool { return e.snd.count() == 1 }) }