feat: document the queue, close mutation gaps, plain relay errors
- Document delivery semantics, retries and the queue API in the README and OpenAPI - Add relay.concurrency to the example config - Test that a relay problem never counts against a member, that the longest-waiting delivery goes first, and the error and limit details - Show the plain relay error when there is a single batch
This commit is contained in:
parent
f3c75dc97d
commit
6461e77a95
9 changed files with 83 additions and 10 deletions
15
README.md
15
README.md
|
|
@ -5,6 +5,9 @@ A modern, single-binary mailing list manager: an evolution of Mailman.
|
|||
- Postfix delivers to parsimony over LMTP; parsimony relays through SMTP.
|
||||
- Discussion lists with member-only posting, `List-*` headers, one-click unsubscribe
|
||||
and DMARC-safe `From` rewriting.
|
||||
- Durable outbound queue: a post is accepted once it is stored, one delivery per recipient,
|
||||
and a background worker sends it with retries (1m, 5m, 15m, 1h, 3h, then every 6h, for up to
|
||||
3 days). Relay outages and restarts lose nothing.
|
||||
- Self-service subscribe and unsubscribe by email with double opt-in:
|
||||
send to `list+join@domain` or `list+leave@domain`, then reply to the confirmation.
|
||||
- Moderated lists: posts wait in a queue that the API can approve or discard.
|
||||
|
|
@ -42,6 +45,18 @@ the unsubscribe links in every message use it. Keep the `api` listener private.
|
|||
curl -H "Authorization: Bearer $PARSIMONY_API_TOKEN" -d '{"name":"dev","domain":"example.org","title":"Dev"}' localhost:8080/api/v1/lists
|
||||
```
|
||||
|
||||
## Delivery and the queue
|
||||
|
||||
LMTP answers `250` when the post is in the queue, so a down relay never bounces mail back to Postfix.
|
||||
It answers `451` only when the database itself fails.
|
||||
|
||||
- Temporary relay errors (4xx, timeouts, connection failures) are retried with backoff.
|
||||
- A permanent rejection (5xx) drops that delivery. Only a bad-mailbox reply (`5.1.x`) counts as a hard
|
||||
bounce for the member; errors such as "relay access denied" never disable members.
|
||||
- Delivery is at-least-once: a crash between the relay accepting a message and the queue entry being
|
||||
removed can send one message to one recipient twice.
|
||||
- `GET /api/v1/queue` and the Queue page show what is pending or failing, and a stuck delivery can be discarded.
|
||||
|
||||
## Admin UI
|
||||
|
||||
The UI is a single-page app in `admin/`, built into `internal/adminui/dist` and embedded in the binary.
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ describe('queue page', () => {
|
|||
await setup([])
|
||||
expect(await screen.findByText(/the queue is empty/i)).toBeInTheDocument()
|
||||
expect(screen.getByText('Pending').nextElementSibling).toHaveTextContent('0')
|
||||
expect(screen.getByText('Oldest').nextElementSibling).toHaveTextContent('—')
|
||||
})
|
||||
|
||||
it('shows totals and each delivery, failing ones with their error', async () => {
|
||||
|
|
@ -54,6 +55,19 @@ describe('queue page', () => {
|
|||
expect(screen.getByText('Oldest').nextElementSibling).toHaveTextContent('2030-01-01 00:00 UTC')
|
||||
})
|
||||
|
||||
it('says how many deliveries are not listed when the server caps the list', async () => {
|
||||
const state = installFakeApi(newState({ queue: [item(1), item(2), item(3)], queueLimit: 2 }))
|
||||
renderApp('/queue', { token: 'good' })
|
||||
expect(await screen.findByText(/showing 2 of 3 deliveries/i)).toBeInTheDocument()
|
||||
expect(state.queue).toHaveLength(3)
|
||||
})
|
||||
|
||||
it('does not mention a cap when everything is listed', async () => {
|
||||
await setup([item(1), item(2)])
|
||||
await screen.findByText('user1@example.com')
|
||||
expect(screen.queryByText(/showing/i)).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('refreshes on demand', async () => {
|
||||
const { user, state } = await setup([item(1)])
|
||||
await screen.findByText('user1@example.com')
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ export interface FakeState {
|
|||
held: Record<number, Held[]>
|
||||
archive: Record<number, Message[]>
|
||||
queue: QueueItem[]
|
||||
/** Like the server, the queue listing is capped. */
|
||||
queueLimit: number
|
||||
nextId: number
|
||||
/** Every request seen, as "METHOD /path". */
|
||||
calls: string[]
|
||||
|
|
@ -17,7 +19,7 @@ export interface FakeState {
|
|||
}
|
||||
|
||||
export function newState(overrides: Partial<FakeState> = {}): FakeState {
|
||||
return { token: 'good', lists: [], members: {}, held: {}, archive: {}, queue: [], nextId: 100, calls: [], fail: {}, ...overrides }
|
||||
return { token: 'good', lists: [], members: {}, held: {}, archive: {}, queue: [], queueLimit: 100, nextId: 100, calls: [], fail: {}, ...overrides }
|
||||
}
|
||||
|
||||
export function aList(over: Partial<List> = {}): List {
|
||||
|
|
@ -64,7 +66,7 @@ export function installFakeApi(state: FakeState) {
|
|||
let m: RegExpMatchArray | null
|
||||
|
||||
if (path === '/queue' && method === 'GET') {
|
||||
const items = [...state.queue].sort((a, b) => b.attempts - a.attempts)
|
||||
const items = [...state.queue].sort((a, b) => b.attempts - a.attempts).slice(0, state.queueLimit)
|
||||
const retrying = state.queue.filter((i) => i.attempts > 0).length
|
||||
const oldest = state.queue.map((i) => i.created_at).sort()[0] ?? null
|
||||
return json(200, { pending: state.queue.length, retrying, oldest, items })
|
||||
|
|
|
|||
|
|
@ -494,13 +494,13 @@ func TestQueueShowsPendingAndFailingDeliveries(t *testing.T) {
|
|||
func TestQueueListIsCapped(t *testing.T) {
|
||||
h, s := queueServer(t)
|
||||
var items []store.NewOutbound
|
||||
for i := 0; i < queueListLimit+20; i++ {
|
||||
for i := 0; i < 120; i++ {
|
||||
items = append(items, store.NewOutbound{From: "b@x.org", Rcpt: fmt.Sprintf("u%d@x.com", i)})
|
||||
}
|
||||
s.Enqueue([]byte("x"), items)
|
||||
var q queueJSON
|
||||
decode(t, do(t, h, "GET", "/api/v1/queue", "", true), &q)
|
||||
if q.Pending != queueListLimit+20 || len(q.Items) != queueListLimit {
|
||||
if q.Pending != 120 || len(q.Items) != 100 {
|
||||
t.Fatalf("pending=%d items=%d", q.Pending, len(q.Items))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -37,6 +37,9 @@ func (r *SMTPRelay) Send(ctx context.Context, from string, to []string, data []b
|
|||
for start := 0; start < len(to); start += size {
|
||||
end := min(start+size, len(to))
|
||||
if err := r.sendBatch(ctx, from, to[start:end], data); err != nil {
|
||||
if len(to) <= size {
|
||||
return err // a single batch: the range adds nothing
|
||||
}
|
||||
return fmt.Errorf("batch %d-%d: %w", start, end, err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import (
|
|||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
|
@ -238,8 +239,8 @@ func TestSendStopsWhenTheContextIsCancelled(t *testing.T) {
|
|||
cancel()
|
||||
select {
|
||||
case err := <-errc:
|
||||
if err == nil {
|
||||
t.Fatal("want an error")
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("want context.Canceled, got %v", err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("Send ignored the cancelled context while talking to a silent relay")
|
||||
|
|
@ -252,8 +253,8 @@ func TestSendHonoursTheContextDeadline(t *testing.T) {
|
|||
defer cancel()
|
||||
start := time.Now()
|
||||
err := r.Send(ctx, "b@x.org", []string{"a@x.com"}, []byte("x"))
|
||||
if err == nil {
|
||||
t.Fatal("want an error")
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("want context.DeadlineExceeded, got %v", err)
|
||||
}
|
||||
if took := time.Since(start); took > 2*time.Second {
|
||||
t.Fatalf("took %v", took)
|
||||
|
|
@ -262,3 +263,19 @@ func TestSendHonoursTheContextDeadline(t *testing.T) {
|
|||
t.Fatalf("a timeout is temporary: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSingleBatchErrorsAreNotDecoratedWithBatchNumbers(t *testing.T) {
|
||||
r := &SMTPRelay{Addr: "127.0.0.1:1"}
|
||||
err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, []byte("x"))
|
||||
if err == nil || strings.HasPrefix(err.Error(), "batch") {
|
||||
t.Fatalf("one recipient, one batch: the error should be the plain cause, got %v", err)
|
||||
}
|
||||
many := make([]string, 120)
|
||||
for i := range many {
|
||||
many[i] = fmt.Sprintf("u%d@x.com", i)
|
||||
}
|
||||
err = (&SMTPRelay{Addr: "127.0.0.1:1", BatchSize: 50}).Send(context.Background(), "b@x.org", many, []byte("x"))
|
||||
if err == nil || !strings.HasPrefix(err.Error(), "batch 0-50:") {
|
||||
t.Fatalf("with several batches the failing range is useful: %v", err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -240,6 +240,27 @@ func TestBadMailboxCountsAHardBounceAndDisablesAfterThree(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestOtherPermanentRejectionsNeverCountAgainstTheMember(t *testing.T) {
|
||||
// A misconfigured relay that refuses everything must not empty the list.
|
||||
e := setup(t)
|
||||
l, _ := e.s.CreateList("dev", "example.org", "Dev")
|
||||
e.s.AddMember(l.ID, "ok@x.com", "")
|
||||
ms, _ := e.s.Members(l.ID)
|
||||
e.snd.fn = func(context.Context, string, []string) error {
|
||||
return &smtp.SMTPError{Code: 554, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "relay access denied"}
|
||||
}
|
||||
for i := 0; i < 5; i++ {
|
||||
e.s.Enqueue([]byte("x"), []store.NewOutbound{{From: "b@x.org", Rcpt: "ok@x.com", MemberID: ms[0].ID}})
|
||||
st, _ := e.w.RunOnce(context.Background())
|
||||
if st.Dropped != 1 || st.Bounced != 0 {
|
||||
t.Fatalf("round %d: %+v", i, st)
|
||||
}
|
||||
}
|
||||
if m, _ := e.s.MemberByID(ms[0].ID); m.BounceCount != 0 || m.Disabled {
|
||||
t.Fatalf("member was punished for a relay problem: %+v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBadMailboxForSystemMailOrUnknownMemberIsHarmless(t *testing.T) {
|
||||
e := setup(t)
|
||||
e.snd.fn = func(context.Context, string, []string) error {
|
||||
|
|
|
|||
|
|
@ -117,10 +117,10 @@ func TestDueIsOrderedByNextAttempt(t *testing.T) {
|
|||
s.Now = func() time.Time { return now }
|
||||
s.Enqueue([]byte("body"), items(3))
|
||||
due, _ := s.DueOutbox(10)
|
||||
s.RetryOutbox(due[0].ID, now.Add(-time.Hour), "e") // oldest next_at
|
||||
s.RetryOutbox(due[2].ID, now.Add(-time.Hour), "e") // the LAST id, but the longest waiting
|
||||
now = now.Add(time.Minute)
|
||||
got, _ := s.DueOutbox(10)
|
||||
if got[0].ID != due[0].ID {
|
||||
if got[0].ID != due[2].ID {
|
||||
t.Fatalf("the longest waiting item goes first: %+v", got)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ lmtp:
|
|||
addr: 0.0.0.0:2424
|
||||
relay:
|
||||
addr: postfix:25
|
||||
concurrency: 4 # simultaneous deliveries
|
||||
web:
|
||||
listen: 0.0.0.0:8081 # public archive and unsubscribe pages; "" disables them
|
||||
api:
|
||||
|
|
|
|||
Loading…
Reference in a new issue