diff --git a/README.md b/README.md index 795a466..59daa1d 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,9 @@ A modern, single-binary mailing list manager: an evolution of Mailman. - Postfix delivers to parsimony over LMTP; parsimony relays through SMTP. - Discussion lists with member-only posting, `List-*` headers, one-click unsubscribe and DMARC-safe `From` rewriting. +- Durable outbound queue: a post is accepted once it is stored, one delivery per recipient, + and a background worker sends it with retries (1m, 5m, 15m, 1h, 3h, then every 6h, for up to + 3 days). Relay outages and restarts lose nothing. - Self-service subscribe and unsubscribe by email with double opt-in: send to `list+join@domain` or `list+leave@domain`, then reply to the confirmation. - Moderated lists: posts wait in a queue that the API can approve or discard. @@ -42,6 +45,18 @@ the unsubscribe links in every message use it. Keep the `api` listener private. curl -H "Authorization: Bearer $PARSIMONY_API_TOKEN" -d '{"name":"dev","domain":"example.org","title":"Dev"}' localhost:8080/api/v1/lists ``` +## Delivery and the queue + +LMTP answers `250` when the post is in the queue, so a down relay never bounces mail back to Postfix. +It answers `451` only when the database itself fails. + +- Temporary relay errors (4xx, timeouts, connection failures) are retried with backoff. +- A permanent rejection (5xx) drops that delivery. Only a bad-mailbox reply (`5.1.x`) counts as a hard + bounce for the member; errors such as "relay access denied" never disable members. +- Delivery is at-least-once: a crash between the relay accepting a message and the queue entry being + removed can send one message to one recipient twice. +- `GET /api/v1/queue` and the Queue page show what is pending or failing, and a stuck delivery can be discarded. + ## Admin UI The UI is a single-page app in `admin/`, built into `internal/adminui/dist` and embedded in the binary. diff --git a/admin/src/pages/QueuePage.test.tsx b/admin/src/pages/QueuePage.test.tsx index 4c65104..9922026 100644 --- a/admin/src/pages/QueuePage.test.tsx +++ b/admin/src/pages/QueuePage.test.tsx @@ -35,6 +35,7 @@ describe('queue page', () => { await setup([]) expect(await screen.findByText(/the queue is empty/i)).toBeInTheDocument() expect(screen.getByText('Pending').nextElementSibling).toHaveTextContent('0') + expect(screen.getByText('Oldest').nextElementSibling).toHaveTextContent('—') }) it('shows totals and each delivery, failing ones with their error', async () => { @@ -54,6 +55,19 @@ describe('queue page', () => { expect(screen.getByText('Oldest').nextElementSibling).toHaveTextContent('2030-01-01 00:00 UTC') }) + it('says how many deliveries are not listed when the server caps the list', async () => { + const state = installFakeApi(newState({ queue: [item(1), item(2), item(3)], queueLimit: 2 })) + renderApp('/queue', { token: 'good' }) + expect(await screen.findByText(/showing 2 of 3 deliveries/i)).toBeInTheDocument() + expect(state.queue).toHaveLength(3) + }) + + it('does not mention a cap when everything is listed', async () => { + await setup([item(1), item(2)]) + await screen.findByText('user1@example.com') + expect(screen.queryByText(/showing/i)).not.toBeInTheDocument() + }) + it('refreshes on demand', async () => { const { user, state } = await setup([item(1)]) await screen.findByText('user1@example.com') diff --git a/admin/src/test/fakeApi.ts b/admin/src/test/fakeApi.ts index 820e9ac..4355086 100644 --- a/admin/src/test/fakeApi.ts +++ b/admin/src/test/fakeApi.ts @@ -9,6 +9,8 @@ export interface FakeState { held: Record archive: Record queue: QueueItem[] + /** Like the server, the queue listing is capped. */ + queueLimit: number nextId: number /** Every request seen, as "METHOD /path". */ calls: string[] @@ -17,7 +19,7 @@ export interface FakeState { } export function newState(overrides: Partial = {}): FakeState { - return { token: 'good', lists: [], members: {}, held: {}, archive: {}, queue: [], nextId: 100, calls: [], fail: {}, ...overrides } + return { token: 'good', lists: [], members: {}, held: {}, archive: {}, queue: [], queueLimit: 100, nextId: 100, calls: [], fail: {}, ...overrides } } export function aList(over: Partial = {}): List { @@ -64,7 +66,7 @@ export function installFakeApi(state: FakeState) { let m: RegExpMatchArray | null if (path === '/queue' && method === 'GET') { - const items = [...state.queue].sort((a, b) => b.attempts - a.attempts) + const items = [...state.queue].sort((a, b) => b.attempts - a.attempts).slice(0, state.queueLimit) const retrying = state.queue.filter((i) => i.attempts > 0).length const oldest = state.queue.map((i) => i.created_at).sort()[0] ?? null return json(200, { pending: state.queue.length, retrying, oldest, items }) diff --git a/internal/api/api_test.go b/internal/api/api_test.go index 6a68261..32f21ba 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -494,13 +494,13 @@ func TestQueueShowsPendingAndFailingDeliveries(t *testing.T) { func TestQueueListIsCapped(t *testing.T) { h, s := queueServer(t) var items []store.NewOutbound - for i := 0; i < queueListLimit+20; i++ { + for i := 0; i < 120; i++ { items = append(items, store.NewOutbound{From: "b@x.org", Rcpt: fmt.Sprintf("u%d@x.com", i)}) } s.Enqueue([]byte("x"), items) var q queueJSON decode(t, do(t, h, "GET", "/api/v1/queue", "", true), &q) - if q.Pending != queueListLimit+20 || len(q.Items) != queueListLimit { + if q.Pending != 120 || len(q.Items) != 100 { t.Fatalf("pending=%d items=%d", q.Pending, len(q.Items)) } } diff --git a/internal/outbound/outbound.go b/internal/outbound/outbound.go index b53088d..9b1e4fc 100644 --- a/internal/outbound/outbound.go +++ b/internal/outbound/outbound.go @@ -37,6 +37,9 @@ func (r *SMTPRelay) Send(ctx context.Context, from string, to []string, data []b for start := 0; start < len(to); start += size { end := min(start+size, len(to)) if err := r.sendBatch(ctx, from, to[start:end], data); err != nil { + if len(to) <= size { + return err // a single batch: the range adds nothing + } return fmt.Errorf("batch %d-%d: %w", start, end, err) } } diff --git a/internal/outbound/outbound_test.go b/internal/outbound/outbound_test.go index 47b0566..b845890 100644 --- a/internal/outbound/outbound_test.go +++ b/internal/outbound/outbound_test.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net" + "strings" "sync" "testing" "time" @@ -238,8 +239,8 @@ func TestSendStopsWhenTheContextIsCancelled(t *testing.T) { cancel() select { case err := <-errc: - if err == nil { - t.Fatal("want an error") + if !errors.Is(err, context.Canceled) { + t.Fatalf("want context.Canceled, got %v", err) } case <-time.After(2 * time.Second): t.Fatal("Send ignored the cancelled context while talking to a silent relay") @@ -252,8 +253,8 @@ func TestSendHonoursTheContextDeadline(t *testing.T) { defer cancel() start := time.Now() err := r.Send(ctx, "b@x.org", []string{"a@x.com"}, []byte("x")) - if err == nil { - t.Fatal("want an error") + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("want context.DeadlineExceeded, got %v", err) } if took := time.Since(start); took > 2*time.Second { t.Fatalf("took %v", took) @@ -262,3 +263,19 @@ func TestSendHonoursTheContextDeadline(t *testing.T) { t.Fatalf("a timeout is temporary: %v", err) } } + +func TestSingleBatchErrorsAreNotDecoratedWithBatchNumbers(t *testing.T) { + r := &SMTPRelay{Addr: "127.0.0.1:1"} + err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, []byte("x")) + if err == nil || strings.HasPrefix(err.Error(), "batch") { + t.Fatalf("one recipient, one batch: the error should be the plain cause, got %v", err) + } + many := make([]string, 120) + for i := range many { + many[i] = fmt.Sprintf("u%d@x.com", i) + } + err = (&SMTPRelay{Addr: "127.0.0.1:1", BatchSize: 50}).Send(context.Background(), "b@x.org", many, []byte("x")) + if err == nil || !strings.HasPrefix(err.Error(), "batch 0-50:") { + t.Fatalf("with several batches the failing range is useful: %v", err) + } +} diff --git a/internal/queue/queue_test.go b/internal/queue/queue_test.go index 69cc74a..fbddf9b 100644 --- a/internal/queue/queue_test.go +++ b/internal/queue/queue_test.go @@ -240,6 +240,27 @@ func TestBadMailboxCountsAHardBounceAndDisablesAfterThree(t *testing.T) { } } +func TestOtherPermanentRejectionsNeverCountAgainstTheMember(t *testing.T) { + // A misconfigured relay that refuses everything must not empty the list. + e := setup(t) + l, _ := e.s.CreateList("dev", "example.org", "Dev") + e.s.AddMember(l.ID, "ok@x.com", "") + ms, _ := e.s.Members(l.ID) + e.snd.fn = func(context.Context, string, []string) error { + return &smtp.SMTPError{Code: 554, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "relay access denied"} + } + for i := 0; i < 5; i++ { + e.s.Enqueue([]byte("x"), []store.NewOutbound{{From: "b@x.org", Rcpt: "ok@x.com", MemberID: ms[0].ID}}) + st, _ := e.w.RunOnce(context.Background()) + if st.Dropped != 1 || st.Bounced != 0 { + t.Fatalf("round %d: %+v", i, st) + } + } + if m, _ := e.s.MemberByID(ms[0].ID); m.BounceCount != 0 || m.Disabled { + t.Fatalf("member was punished for a relay problem: %+v", m) + } +} + func TestBadMailboxForSystemMailOrUnknownMemberIsHarmless(t *testing.T) { e := setup(t) e.snd.fn = func(context.Context, string, []string) error { diff --git a/internal/store/outbox_test.go b/internal/store/outbox_test.go index 9e88886..4ec14bd 100644 --- a/internal/store/outbox_test.go +++ b/internal/store/outbox_test.go @@ -117,10 +117,10 @@ func TestDueIsOrderedByNextAttempt(t *testing.T) { s.Now = func() time.Time { return now } s.Enqueue([]byte("body"), items(3)) due, _ := s.DueOutbox(10) - s.RetryOutbox(due[0].ID, now.Add(-time.Hour), "e") // oldest next_at + s.RetryOutbox(due[2].ID, now.Add(-time.Hour), "e") // the LAST id, but the longest waiting now = now.Add(time.Minute) got, _ := s.DueOutbox(10) - if got[0].ID != due[0].ID { + if got[0].ID != due[2].ID { t.Fatalf("the longest waiting item goes first: %+v", got) } } diff --git a/parsimony.example.yaml b/parsimony.example.yaml index baea5be..d5c493b 100644 --- a/parsimony.example.yaml +++ b/parsimony.example.yaml @@ -5,6 +5,7 @@ lmtp: addr: 0.0.0.0:2424 relay: addr: postfix:25 + concurrency: 4 # simultaneous deliveries web: listen: 0.0.0.0:8081 # public archive and unsubscribe pages; "" disables them api: