Loading…
} + {posts.isError &&{posts.error.message}
} + {posts.data && items.length === 0 && ( +Nothing has been sent to this list yet.
+ )} + {items.length > 0 && ( +", []byte("Message-ID:
\r\nSubject: hi\r\n\r\nbody\r\n"), items); err != nil { + t.Fatal(err) + } + return l +} + +func (e *env) onlyPost(t *testing.T, l store.List) store.Post { + t.Helper() + posts, err := e.s.ListPosts(l.ID, 0) + if err != nil || len(posts) != 1 { + t.Fatalf("posts: %+v %v", posts, err) + } + return posts[0] +} + +func TestTheWorkerCountsEachOutcomeOfAPost(t *testing.T) { + e := setup(t) + l := e.post(t, 4) + e.snd.fn = func(_ context.Context, _ string, to []string) error { + switch to[0] { + case "b@x.com": + return errors.New("connection refused") + case "c@x.com": + return &smtp.SMTPError{Code: 554, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "policy"} + case "d@x.com": + return outbound.AtRcpt(&smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"}) + } + return nil + } + st, err := e.w.RunOnce(context.Background()) + if err != nil || st.Sent != 1 || st.Retried != 1 || st.Failed != 1 || st.Dropped != 1 { + t.Fatalf("run: %+v %v", st, err) + } + p := e.onlyPost(t, l) + if p.Recipients != 4 || p.Sent != 1 || p.Failed != 1 || p.Dropped != 1 || p.Pending != 1 { + t.Fatalf("post: %+v", p) + } + // The retried delivery goes out later. + e.snd.fn = nil + e.clk.set(t0.Add(Backoff(1))) + e.w.RunOnce(context.Background()) + if p := e.onlyPost(t, l); p.Sent != 2 || p.Pending != 0 { + t.Fatalf("post: %+v", p) + } +} + +func TestGivingUpAfterMaxAgeCountsAsFailed(t *testing.T) { + e := setup(t) + l := e.post(t, 1) + e.snd.fn = func(context.Context, string, []string) error { return errors.New("down") } + e.clk.set(t0.Add(MaxAge + 1)) + if st, _ := e.w.RunOnce(context.Background()); st.Failed != 1 { + t.Fatalf("stats: %+v", st) + } + if p := e.onlyPost(t, l); p.Failed != 1 || p.Pending != 0 { + t.Fatalf("post: %+v", p) + } +} + +func TestAMissingBodyCountsAsDropped(t *testing.T) { + e := setup(t) + e.w.readBody = func(int64) ([]byte, error) { return nil, store.ErrNotFound } + l := e.post(t, 2) + e.w.RunOnce(context.Background()) + if p := e.onlyPost(t, l); p.Dropped != 2 || p.Sent != 0 || p.Pending != 0 { + t.Fatalf("post: %+v", p) + } +} + +func TestDiscussionMailIsSentBeforeABigNewsletterQueuedEarlier(t *testing.T) { + e := setup(t) + e.w.Concurrency = 1 + var news []store.NewOutbound + for i := 0; i < 120; i++ { + news = append(news, store.NewOutbound{From: "b@x.org", Rcpt: "reader@x.com", Priority: 1}) + } + if err := e.s.Enqueue([]byte("Subject: newsletter\r\n\r\nx"), news); err != nil { + t.Fatal(err) + } + e.clk.set(t0.Add(1)) + e.s.Enqueue([]byte("Subject: talk\r\n\r\nx"), []store.NewOutbound{ + {From: "b@x.org", Rcpt: "first@x.com"}, {From: "b@x.org", Rcpt: "second@x.com"}, + }) + if _, err := e.w.RunOnce(context.Background()); err != nil { + t.Fatal(err) + } + if e.snd.sent[0].To[0] != "first@x.com" || e.snd.sent[1].To[0] != "second@x.com" || e.snd.sent[2].To[0] != "reader@x.com" { + t.Fatalf("order: %v %v %v", e.snd.sent[0].To, e.snd.sent[1].To, e.snd.sent[2].To) + } +} diff --git a/internal/queue/queue.go b/internal/queue/queue.go index 9976a9c..de5a634 100644 --- a/internal/queue/queue.go +++ b/internal/queue/queue.go @@ -60,6 +60,15 @@ type Worker struct { // Sign, when set, signs the final message (DKIM). On error the message is sent unsigned: // not delivering is worse than a missed signature. Sign func(msg []byte) ([]byte, error) + // Rate is the most sends started per second, for relays that throttle (Amazon SES). Zero or + // negative means unlimited. It is read once, at the first send. + Rate float64 + + pacerOnce sync.Once + pacer *pacer + // Hooks for tests; nil means the real clock and sleep. + pacerNow func() time.Time + pacerSleep func(ctx context.Context, d time.Duration) error wakeOnce sync.Once wake chan struct{} @@ -219,7 +228,7 @@ func (w *Worker) RunOnce(ctx context.Context) (Stats, error) { func (w *Worker) deliver(ctx context.Context, cache *bodies, it store.OutboxItem) Stats { data, err := cache.get(it.MsgID) if errors.Is(err, store.ErrNotFound) { - w.complete(it) + w.drop(it) return Stats{Dropped: 1} } if err != nil { @@ -240,6 +249,10 @@ func (w *Worker) deliver(ctx context.Context, cache *bodies, it store.OutboxItem if timeout <= 0 { timeout = defaultTimeout } + // Wait for the slot before starting the timeout, so waiting does not eat into the send. + if err := w.pace().Wait(ctx); err != nil { + return Stats{} // shutting down: the delivery stays queued, unchanged + } sctx, cancel := context.WithTimeout(ctx, timeout) err = w.Sender.Send(sctx, it.From, []string{it.Rcpt}, msg) cancel() @@ -259,7 +272,7 @@ func (w *Worker) deliver(ctx context.Context, cache *bodies, it store.OutboxItem } } log.Printf("queue: drop %s: %v", it.Rcpt, err) - w.complete(it) + w.drop(it) return out case outbound.IsPermanent(err): // The relay refused the message itself (policy, size, line length...): nobody gets it, so @@ -277,6 +290,21 @@ func (w *Worker) deliver(ctx context.Context, cache *bodies, it store.OutboxItem return Stats{Retried: 1} } +// pace returns the pacer in front of every send; nil (never waits) when Rate is not set. +func (w *Worker) pace() *pacer { + w.pacerOnce.Do(func() { + now, sleep := w.pacerNow, w.pacerSleep + if now == nil { + now = time.Now + } + if sleep == nil { + sleep = sleepCtx + } + w.pacer = newPacer(w.Rate, now, sleep) + }) + return w.pacer +} + // giveUp marks a delivery as failed for good and logs it. func (w *Worker) giveUp(it store.OutboxItem, reason string) Stats { log.Printf("queue: %s: %s", it.Rcpt, reason) @@ -292,3 +320,10 @@ func (w *Worker) complete(it store.OutboxItem) { log.Printf("queue: remove %d: %v", it.ID, err) } } + +// drop removes a delivery that will never be made; a delivery of a post counts as dropped. +func (w *Worker) drop(it store.OutboxItem) { + if err := w.Store.DropOutbox(it.ID); err != nil && !errors.Is(err, store.ErrNotFound) { + log.Printf("queue: drop %d: %v", it.ID, err) + } +} diff --git a/internal/queue/queue_test.go b/internal/queue/queue_test.go index 7e2dbd2..24ae823 100644 --- a/internal/queue/queue_test.go +++ b/internal/queue/queue_test.go @@ -668,3 +668,91 @@ func TestAMissingBodyIsDroppedForEveryRecipientWithOneRead(t *testing.T) { t.Fatalf("%d reads", n) } } + +func TestRateSpacesTheSendsOfAWorker(t *testing.T) { + e := setup(t) + e.enqueue(t, 5) + e.w.Rate = 2 // one send every 500 ms + f := &fakeTime{t: t0} + e.w.pacerNow, e.w.pacerSleep = f.now, f.sleep + st, _ := e.w.RunOnce(context.Background()) + if st.Sent != 5 { + t.Fatalf("%+v", st) + } + got := f.durations() // 5 deliveries: slots at 0, 1, 2, 3 and 4 intervals + want := []time.Duration{500 * time.Millisecond, time.Second, 1500 * time.Millisecond, 2 * time.Second} + if len(got) != len(want) { + t.Fatalf("sleeps %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("sleeps %v, want %v", got, want) + } + } +} + +func TestRateOfZeroNeverWaits(t *testing.T) { + e := setup(t) + e.enqueue(t, 5) + f := &fakeTime{t: t0} + e.w.pacerNow, e.w.pacerSleep = f.now, f.sleep + e.w.RunOnce(context.Background()) + if len(f.slept) != 0 || e.snd.count() != 5 { + t.Fatalf("slept %v, sent %d", f.slept, e.snd.count()) + } +} + +func TestEverySendAttemptIsPacedEvenARetry(t *testing.T) { + e := setup(t) + e.enqueue(t, 1) + e.w.Rate = 2 + f := &fakeTime{t: t0} + e.w.pacerNow, e.w.pacerSleep = f.now, f.sleep + e.snd.fn = func(context.Context, string, []string) error { return errors.New("try later") } + e.w.RunOnce(context.Background()) + e.clk.set(t0.Add(time.Hour)) + e.w.RunOnce(context.Background()) + if e.snd.calls != 2 || len(f.slept) != 1 { + t.Fatalf("calls=%d sleeps=%v: the second attempt shares the pacer with the first", e.snd.calls, f.slept) + } +} + +func TestPacingIsCancelledWithoutRecordingAnAttempt(t *testing.T) { + e := setup(t) + e.enqueue(t, 2) + e.w.Rate = 1 + e.w.Concurrency = 1 + ctx, cancel := context.WithCancel(context.Background()) + f := &fakeTime{t: t0} + e.w.pacerNow = f.now + e.w.pacerSleep = func(ctx context.Context, d time.Duration) error { + cancel() // shutdown arrives while the delivery waits for its slot + return ctx.Err() + } + st, _ := e.w.RunOnce(ctx) + if st != (Stats{Sent: 1}) { + t.Fatalf("stats %+v: the delivery that did not get its slot is not counted", st) + } + list, _ := e.s.ListOutbox(10) + if len(list) != 1 || list[0].Attempts != 0 || list[0].LastError != "" { + t.Fatalf("a shutdown while waiting is not a delivery failure: %+v", list) + } + if e.snd.calls != 1 { + t.Fatalf("calls = %d: only the first delivery, which did not wait, was sent", e.snd.calls) + } +} + +func TestRateHoldsInRealTime(t *testing.T) { + e := setup(t) + e.enqueue(t, 20) + e.w.Rate = 200 // 5 ms apart: 19 intervals take 95 ms + start := time.Now() + st, _ := e.w.RunOnce(context.Background()) + took := time.Since(start) + if st.Sent != 20 { + t.Fatalf("%+v", st) + } + if took < 90*time.Millisecond || took > time.Second { + t.Fatalf("20 sends at 200/s took %v, want about 95 ms", took) + } +} diff --git a/internal/router/commands.go b/internal/router/commands.go index bd8ab3d..b8c621a 100644 --- a/internal/router/commands.go +++ b/internal/router/commands.go @@ -94,6 +94,24 @@ func (r *Router) join(l store.List, sender string) error { return r.askConfirmation(l, sender, store.ActionJoin) } +// RequestJoin does what the +join command does for addr, for a caller that is not an LMTP session +// (the public subscribe form): there is no envelope sender to check, so the caller must have +// validated addr. It returns nil whenever nothing should be revealed: an unknown or closed list, an +// active member, a request the store rate limits. It returns an error only for internal failures. +func (r *Router) RequestJoin(listAddress, addr string) error { + l, err := r.store.FindList(listAddress) + if errors.Is(err, store.ErrNotFound) { + return nil + } + if err != nil { + return err + } + if err := r.join(l, addr); err != nil && !errors.Is(err, ErrJoinClosed) { + return err + } + return nil +} + func (r *Router) leave(l store.List, sender string) error { if ok, err := r.store.IsMember(l.ID, sender); err != nil || !ok { return err diff --git a/internal/router/m2_test.go b/internal/router/m2_test.go index ab9d060..92c96c7 100644 --- a/internal/router/m2_test.go +++ b/internal/router/m2_test.go @@ -628,3 +628,76 @@ func TestDroppingARCHeadersKeepsSimilarlyNamedOnes(t *testing.T) { t.Errorf("only the ARC- prefix is dropped: %v", h) } } + +func TestRequestJoinQueuesTheConfirmationMail(t *testing.T) { + e := newEnv(t) + if err := e.r.RequestJoin("dev@example.org", "carol@x.com"); err != nil { + t.Fatal(err) + } + e.flush() + m := lastMail(t, e.out) + if len(m.To) != 1 || m.To[0] != "carol@x.com" { + t.Fatalf("confirmation must go to the requester: %+v", m.To) + } + if ok, _ := e.s.IsMember(e.list.ID, "carol@x.com"); ok { + t.Fatal("not a member before confirming") + } + // The mail is the usual one: replying to it confirms. + if _, err := e.command("carol@x.com", "dev+confirm-"+token(t, m)+"@example.org", nil); err != nil { + t.Fatal(err) + } + if ok, _ := e.s.IsMember(e.list.ID, "carol@x.com"); !ok { + t.Fatal("carol must be a member after confirming") + } +} + +// Nothing may tell the caller whether the list exists, is closed, or the address is known. +func TestRequestJoinRevealsNothing(t *testing.T) { + e := newEnv(t) + closed, _ := e.s.CreateList("closed", "example.org", "Closed") + e.s.SetPolicies(closed.ID, store.PostMembers, store.JoinClosed, store.ArchivePrivate) + for _, c := range []struct{ name, list, addr string }{ + {"unknown list", "nope@example.org", "carol@x.com"}, + {"closed list", "closed@example.org", "carol@x.com"}, + {"active member", "dev@example.org", "alice@example.com"}, + } { + if err := e.r.RequestJoin(c.list, c.addr); err != nil { + t.Errorf("%s: want nil, got %v", c.name, err) + } + } + e.flush() + if len(e.out.Sent) != 0 { + t.Fatalf("no mail for these cases, sent %d", len(e.out.Sent)) + } + // Asking twice is rate limited by the store: still nil, still one mail. + e.r.RequestJoin("dev@example.org", "carol@x.com") + if err := e.r.RequestJoin("dev@example.org", "carol@x.com"); err != nil { + t.Fatalf("rate limited: want nil, got %v", err) + } + e.flush() + if len(e.out.Sent) != 1 { + t.Fatalf("one mail expected, sent %d", len(e.out.Sent)) + } +} + +func TestRequestJoinLetsABounceDisabledMemberRejoin(t *testing.T) { + e := newEnv(t) + for i := 0; i < store.BounceThreshold; i++ { + e.s.RecordHardBounce(e.alice, store.BounceThreshold) + } + if err := e.r.RequestJoin("dev@example.org", "alice@example.com"); err != nil { + t.Fatal(err) + } + e.flush() + if m := lastMail(t, e.out); len(m.To) != 1 || m.To[0] != "alice@example.com" { + t.Fatalf("to = %v", m.To) + } +} + +func TestRequestJoinReportsAnInternalFailure(t *testing.T) { + e := newEnv(t) + e.s.Close() + if err := e.r.RequestJoin("dev@example.org", "carol@x.com"); err == nil { + t.Fatal("a database failure must be reported") + } +} diff --git a/internal/router/priority_test.go b/internal/router/priority_test.go new file mode 100644 index 0000000..ceab018 --- /dev/null +++ b/internal/router/priority_test.go @@ -0,0 +1,113 @@ +package router + +import ( + "testing" + + "codeberg.org/squidd/parsimony/internal/store" +) + +// priorityEnv is newEnv with the publishers post policy, for the delivery priority tests. +func priorityEnv(t *testing.T) *env { + t.Helper() + e := newEnv(t) + if err := e.s.SetPolicies(e.list.ID, store.PostPublishers, store.JoinOpen, store.ArchivePrivate); err != nil { + t.Fatal(err) + } + l, _ := e.s.GetList(e.list.ID) + e.list = l + return e +} + +func duePriorities(t *testing.T, s *store.Store) []int { + t.Helper() + due, err := s.DueOutbox(100) + if err != nil { + t.Fatal(err) + } + var out []int + for _, d := range due { + out = append(out, d.Priority) + } + return out +} + +func allEqual(ps []int, want int, n int) bool { + if len(ps) != n { + return false + } + for _, p := range ps { + if p != want { + return false + } + } + return true +} + +const samplePost = "From: alice@example.com\r\nSubject: hi\r\nMessage-ID:
\r\n\r\nbody\r\n" + +func TestANewsletterIsDeliveredWithPriorityOne(t *testing.T) { + e := priorityEnv(t) + res, err := e.r.prepare(e.list, []byte(samplePost)) + if err != nil { + t.Fatal(err) + } + if err := e.r.Deliver(res); err != nil { + t.Fatal(err) + } + if ps := duePriorities(t, e.s); !allEqual(ps, 1, 2) { + t.Fatalf("priorities: %v", ps) + } +} + +func TestADiscussionPostIsDeliveredWithPriorityZero(t *testing.T) { + e := newEnv(t) + if err := e.post(t, samplePost); err != nil { + t.Fatal(err) + } + if ps := duePriorities(t, e.s); !allEqual(ps, 0, 2) { + t.Fatalf("priorities: %v", ps) + } +} + +func TestAReleasedNewsletterHasPriorityOne(t *testing.T) { + e := priorityEnv(t) + id, _ := e.s.HoldMessage(e.list.ID, "alice@example.com", []byte(samplePost), "r") + if err := e.r.Release(id); err != nil { + t.Fatal(err) + } + if ps := duePriorities(t, e.s); !allEqual(ps, 1, 2) { + t.Fatalf("priorities: %v", ps) + } +} + +func TestAReleasedDiscussionPostHasPriorityZero(t *testing.T) { + e := newEnv(t) + id, _ := e.s.HoldMessage(e.list.ID, "alice@example.com", []byte(samplePost), "r") + if err := e.r.Release(id); err != nil { + t.Fatal(err) + } + if ps := duePriorities(t, e.s); !allEqual(ps, 0, 2) { + t.Fatalf("priorities: %v", ps) + } +} + +func TestConfirmationMailOfANewsletterHasPriorityZero(t *testing.T) { + e := priorityEnv(t) + if handled, err := e.r.Command("carol@x.com", "dev+join@example.org", nil); !handled || err != nil { + t.Fatalf("join: %v %v", handled, err) + } + if ps := duePriorities(t, e.s); !allEqual(ps, 0, 1) { + t.Fatalf("priorities: %v", ps) + } +} + +func TestAPostIsRecordedWithItsRecipients(t *testing.T) { + e := newEnv(t) + if err := e.post(t, samplePost); err != nil { + t.Fatal(err) + } + posts, err := e.s.ListPosts(e.list.ID, 0) + if err != nil || len(posts) != 1 || posts[0].Recipients != 2 || posts[0].MessageID != "
" || posts[0].Subject == "" {
+ t.Fatalf("posts: %+v %v", posts, err)
+ }
+}
diff --git a/internal/router/publisher_test.go b/internal/router/publisher_test.go
new file mode 100644
index 0000000..c80f3db
--- /dev/null
+++ b/internal/router/publisher_test.go
@@ -0,0 +1,143 @@
+package router
+
+import (
+ "errors"
+ "strings"
+ "testing"
+
+ "codeberg.org/squidd/parsimony/internal/store"
+)
+
+// newsletterEnv is the test list with the publishers post policy and alice as its only publisher.
+func newsletterEnv(t *testing.T) *env {
+ t.Helper()
+ e := newEnv(t)
+ if err := e.s.SetPolicies(e.list.ID, store.PostPublishers, store.JoinOpen, store.ArchivePrivate); err != nil {
+ t.Fatal(err)
+ }
+ if err := e.s.SetPublisher(e.list.ID, "alice@example.com", true); err != nil {
+ t.Fatal(err)
+ }
+ return e
+}
+
+// nothingKept checks that a refused post left no trace: not held, not archived, not queued.
+func nothingKept(t *testing.T, e *env) {
+ t.Helper()
+ if held, _ := e.s.ListHeld(e.list.ID); len(held) != 0 {
+ t.Fatalf("held: %+v", held)
+ }
+ if msgs, _, _ := e.s.ListMessages(e.list.ID, 10, 0); len(msgs) != 0 {
+ t.Fatalf("archived: %+v", msgs)
+ }
+ if st, _ := e.s.OutboxStats(); st.Pending != 0 {
+ t.Fatalf("queued: %+v", st)
+ }
+}
+
+func TestAPublisherPostsToANewsletter(t *testing.T) {
+ e := newsletterEnv(t)
+ res, err := e.r.Route("Alice@Example.com", "dev@example.org", postFrom("Alice {{.List.Address}} {{.List.Address}} {{.Error}} We send one message to confirm the address. Nothing happens until you reply to it. If this address can subscribe, a confirmation message is on its way. Open it and reply to confirm. Please try again later. Stop receiving messages from {{.Title}} ({{.Address}})?{{.List.Title}}
Subscribe to {{.List.Title}}
+Check your email
+Too many requests
Unsubscribe