Merge pull request 'Milestone 5: persistent outbound queue with retries' (#14) from feat/m5-outbound-queue into main
This commit is contained in:
commit
eed5d857b3
29 changed files with 2195 additions and 178 deletions
15
README.md
15
README.md
|
|
@ -5,6 +5,9 @@ A modern, single-binary mailing list manager: an evolution of Mailman.
|
|||
- Postfix delivers to parsimony over LMTP; parsimony relays through SMTP.
|
||||
- Discussion lists with member-only posting, `List-*` headers, one-click unsubscribe
|
||||
and DMARC-safe `From` rewriting.
|
||||
- Durable outbound queue: a post is accepted once it is stored, one delivery per recipient,
|
||||
and a background worker sends it with retries (1m, 5m, 15m, 1h, 3h, then every 6h, for up to
|
||||
3 days). Relay outages and restarts lose nothing.
|
||||
- Self-service subscribe and unsubscribe by email with double opt-in:
|
||||
send to `list+join@domain` or `list+leave@domain`, then reply to the confirmation.
|
||||
- Moderated lists: posts wait in a queue that the API can approve or discard.
|
||||
|
|
@ -42,6 +45,18 @@ the unsubscribe links in every message use it. Keep the `api` listener private.
|
|||
curl -H "Authorization: Bearer $PARSIMONY_API_TOKEN" -d '{"name":"dev","domain":"example.org","title":"Dev"}' localhost:8080/api/v1/lists
|
||||
```
|
||||
|
||||
## Delivery and the queue
|
||||
|
||||
LMTP answers `250` when the post is in the queue, so a down relay never bounces mail back to Postfix.
|
||||
It answers `451` only when the database itself fails.
|
||||
|
||||
- Temporary relay errors (4xx, timeouts, connection failures) are retried with backoff.
|
||||
- A permanent rejection (5xx) drops that delivery. Only a bad-mailbox reply (`5.1.x`) counts as a hard
|
||||
bounce for the member; errors such as "relay access denied" never disable members.
|
||||
- Delivery is at-least-once: a crash between the relay accepting a message and the queue entry being
|
||||
removed can send one message to one recipient twice.
|
||||
- `GET /api/v1/queue` and the Queue page show what is pending or failing, and a stuck delivery can be discarded.
|
||||
|
||||
## Admin UI
|
||||
|
||||
The UI is a single-page app in `admin/`, built into `internal/adminui/dist` and embedded in the binary.
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import { ArchiveTab } from '@/tabs/ArchiveTab'
|
|||
import { HeldTab } from '@/tabs/HeldTab'
|
||||
import { MembersTab } from '@/tabs/MembersTab'
|
||||
import { SettingsTab } from '@/tabs/SettingsTab'
|
||||
import { QueuePage } from '@/pages/QueuePage'
|
||||
import { Toaster } from '@/components/ui/sonner'
|
||||
|
||||
/** Routes only: providers (query client, router, auth) are set up by the caller. */
|
||||
|
|
@ -18,6 +19,7 @@ export function App() {
|
|||
<Route path="/login" element={<LoginPage />} />
|
||||
<Route element={<Shell />}>
|
||||
<Route index element={<ListsPage />} />
|
||||
<Route path="queue" element={<QueuePage />} />
|
||||
<Route path="lists/:id" element={<ListPage />}>
|
||||
<Route index element={<SettingsTab />} />
|
||||
<Route path="members" element={<MembersTab />} />
|
||||
|
|
|
|||
|
|
@ -11,9 +11,14 @@ export function Shell() {
|
|||
<div className="min-h-screen">
|
||||
<header className="border-b">
|
||||
<div className="mx-auto flex max-w-5xl items-center justify-between p-4">
|
||||
<Link to="/" className="text-lg font-semibold">
|
||||
Parsimony
|
||||
</Link>
|
||||
<div className="flex items-center gap-6">
|
||||
<Link to="/" className="text-lg font-semibold">
|
||||
Parsimony
|
||||
</Link>
|
||||
<Link to="/queue" className="text-sm text-muted-foreground hover:text-foreground">
|
||||
Queue
|
||||
</Link>
|
||||
</div>
|
||||
<Button variant="outline" size="sm" onClick={logout}>
|
||||
Sign out
|
||||
</Button>
|
||||
|
|
|
|||
|
|
@ -149,4 +149,14 @@ describe('endpoints', () => {
|
|||
['GET', '/api/v1/archive/9'],
|
||||
])
|
||||
})
|
||||
|
||||
it('reads the queue and discards a delivery', async () => {
|
||||
mockFetch(respond(204))
|
||||
await api.queue.discard(4)
|
||||
await api.queue.get().catch(() => undefined)
|
||||
expect(calls.map((c) => [c.init.method, c.url])).toEqual([
|
||||
['DELETE', '/api/v1/queue/4'],
|
||||
['GET', '/api/v1/queue'],
|
||||
])
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -47,6 +47,22 @@ export interface ArchivePage {
|
|||
messages: Message[]
|
||||
}
|
||||
|
||||
export interface QueueItem {
|
||||
id: number
|
||||
recipient: string
|
||||
attempts: number
|
||||
last_error: string
|
||||
next_attempt: string
|
||||
created_at: string
|
||||
}
|
||||
|
||||
export interface Queue {
|
||||
pending: number
|
||||
retrying: number
|
||||
oldest: string | null
|
||||
items: QueueItem[]
|
||||
}
|
||||
|
||||
export class ApiError extends Error {
|
||||
readonly status: number
|
||||
constructor(status: number, message: string) {
|
||||
|
|
@ -130,6 +146,10 @@ export const api = {
|
|||
approve: (id: number) => request<void>('POST', `/held/${id}/approve`),
|
||||
discard: (id: number) => request<void>('DELETE', `/held/${id}`),
|
||||
},
|
||||
queue: {
|
||||
get: () => request<Queue>('GET', '/queue'),
|
||||
discard: (id: number) => request<void>('DELETE', `/queue/${id}`),
|
||||
},
|
||||
archive: {
|
||||
list: (listId: number, p: { q?: string; limit?: number; offset?: number } = {}) => {
|
||||
const qs = new URLSearchParams()
|
||||
|
|
|
|||
127
admin/src/pages/QueuePage.test.tsx
Normal file
127
admin/src/pages/QueuePage.test.tsx
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
import { act, screen, waitFor, within } from '@testing-library/react'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { QueueItem } from '@/lib/api'
|
||||
import { installFakeApi, newState } from '@/test/fakeApi'
|
||||
import { renderApp } from '@/test/render'
|
||||
|
||||
const item = (id: number, over: Partial<QueueItem> = {}): QueueItem => ({
|
||||
id,
|
||||
recipient: `user${id}@example.com`,
|
||||
attempts: 0,
|
||||
last_error: '',
|
||||
next_attempt: '2030-01-02T03:04:05Z',
|
||||
created_at: '2030-01-01T00:00:00Z',
|
||||
...over,
|
||||
})
|
||||
|
||||
async function setup(items: QueueItem[]) {
|
||||
const state = installFakeApi(newState({ queue: items }))
|
||||
const ctx = renderApp('/queue', { token: 'good' })
|
||||
await screen.findByRole('heading', { name: /outbound queue/i })
|
||||
return { ...ctx, state }
|
||||
}
|
||||
|
||||
afterEach(() => vi.useRealTimers())
|
||||
|
||||
describe('queue page', () => {
|
||||
it('is reachable from the header', async () => {
|
||||
installFakeApi(newState())
|
||||
const { user } = renderApp('/', { token: 'good' })
|
||||
await user.click(await screen.findByRole('link', { name: /^queue$/i }))
|
||||
expect(await screen.findByRole('heading', { name: /outbound queue/i })).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('says when the queue is empty', async () => {
|
||||
await setup([])
|
||||
expect(await screen.findByText(/the queue is empty/i)).toBeInTheDocument()
|
||||
expect(screen.getByText('Pending').nextElementSibling).toHaveTextContent('0')
|
||||
expect(screen.getByText('Oldest').nextElementSibling).toHaveTextContent('—')
|
||||
})
|
||||
|
||||
it('shows totals and each delivery, failing ones with their error', async () => {
|
||||
await setup([
|
||||
item(1),
|
||||
item(2, { attempts: 3, last_error: 'connection refused', next_attempt: '2030-01-02T05:00:00Z' }),
|
||||
])
|
||||
const failing = (await screen.findByText('user2@example.com')).closest('tr')!
|
||||
expect(within(failing).getByText('3')).toBeInTheDocument()
|
||||
expect(within(failing).getByText('connection refused')).toBeInTheDocument()
|
||||
expect(within(failing).getByText('2030-01-02 05:00 UTC')).toBeInTheDocument()
|
||||
const fresh = screen.getByText('user1@example.com').closest('tr')!
|
||||
expect(within(fresh).getByText('—')).toBeInTheDocument()
|
||||
|
||||
expect(screen.getByText('Pending').nextElementSibling).toHaveTextContent('2')
|
||||
expect(screen.getByText('Retrying').nextElementSibling).toHaveTextContent('1')
|
||||
expect(screen.getByText('Oldest').nextElementSibling).toHaveTextContent('2030-01-01 00:00 UTC')
|
||||
})
|
||||
|
||||
it('says how many deliveries are not listed when the server caps the list', async () => {
|
||||
const state = installFakeApi(newState({ queue: [item(1), item(2), item(3)], queueLimit: 2 }))
|
||||
renderApp('/queue', { token: 'good' })
|
||||
expect(await screen.findByText(/showing 2 of 3 deliveries/i)).toBeInTheDocument()
|
||||
expect(state.queue).toHaveLength(3)
|
||||
})
|
||||
|
||||
it('does not mention a cap when everything is listed', async () => {
|
||||
await setup([item(1), item(2)])
|
||||
await screen.findByText('user1@example.com')
|
||||
expect(screen.queryByText(/showing/i)).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('refreshes on demand', async () => {
|
||||
const { user, state } = await setup([item(1)])
|
||||
await screen.findByText('user1@example.com')
|
||||
state.queue = []
|
||||
await user.click(screen.getByRole('button', { name: /refresh/i }))
|
||||
expect(await screen.findByText(/the queue is empty/i)).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('refreshes by itself every few seconds', async () => {
|
||||
vi.useFakeTimers({ shouldAdvanceTime: true }) // before the page starts its polling timer
|
||||
const { state } = await setup([item(1)])
|
||||
await screen.findByText('user1@example.com')
|
||||
state.queue = []
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(6000)
|
||||
})
|
||||
expect(await screen.findByText(/the queue is empty/i)).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('shows a loading error', async () => {
|
||||
installFakeApi(newState({ fail: { 'GET /queue': { status: 500, error: 'database is locked' } } }))
|
||||
renderApp('/queue', { token: 'good' })
|
||||
expect(await screen.findByRole('alert')).toHaveTextContent('database is locked')
|
||||
})
|
||||
|
||||
describe('discarding', () => {
|
||||
async function open() {
|
||||
const ctx = await setup([item(1), item(2)])
|
||||
await ctx.user.click(within((await screen.findByText('user1@example.com')).closest('tr')!).getByRole('button', { name: /discard/i }))
|
||||
return { ...ctx, dialog: await screen.findByRole('alertdialog') }
|
||||
}
|
||||
|
||||
it('confirms first, naming the recipient; cancel keeps it', async () => {
|
||||
const { user, state, dialog } = await open()
|
||||
expect(dialog).toHaveTextContent('user1@example.com')
|
||||
await user.click(within(dialog).getByRole('button', { name: /cancel/i }))
|
||||
await waitFor(() => expect(screen.queryByRole('alertdialog')).not.toBeInTheDocument())
|
||||
expect(state.queue).toHaveLength(2)
|
||||
})
|
||||
|
||||
it('discards the delivery', async () => {
|
||||
const { user, state, dialog } = await open()
|
||||
await user.click(within(dialog).getByRole('button', { name: /^discard$/i }))
|
||||
await waitFor(() => expect(screen.queryByText('user1@example.com')).not.toBeInTheDocument())
|
||||
expect(state.queue.map((i) => i.id)).toEqual([2])
|
||||
expect(screen.getByText('user2@example.com')).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('shows a server error inside the dialog', async () => {
|
||||
const { user, state, dialog } = await open()
|
||||
state.fail['DELETE /queue/1'] = { status: 500, error: 'locked' }
|
||||
await user.click(within(dialog).getByRole('button', { name: /^discard$/i }))
|
||||
expect(await within(dialog).findByRole('alert')).toHaveTextContent('locked')
|
||||
expect(state.queue).toHaveLength(2)
|
||||
})
|
||||
})
|
||||
})
|
||||
141
admin/src/pages/QueuePage.tsx
Normal file
141
admin/src/pages/QueuePage.tsx
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
|
||||
import { useState } from 'react'
|
||||
import { toast } from 'sonner'
|
||||
import {
|
||||
AlertDialog,
|
||||
AlertDialogCancel,
|
||||
AlertDialogContent,
|
||||
AlertDialogDescription,
|
||||
AlertDialogFooter,
|
||||
AlertDialogHeader,
|
||||
AlertDialogTitle,
|
||||
} from '@/components/ui/alert-dialog'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { Card, CardContent } from '@/components/ui/card'
|
||||
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table'
|
||||
import { api, type QueueItem } from '@/lib/api'
|
||||
import { formatDate } from '@/lib/format'
|
||||
|
||||
/** How often the page reloads by itself. */
|
||||
export const REFRESH_MS = 5000
|
||||
|
||||
export function QueuePage() {
|
||||
const queue = useQuery({ queryKey: ['queue'], queryFn: api.queue.get, refetchInterval: REFRESH_MS })
|
||||
const [discarding, setDiscarding] = useState<QueueItem | null>(null)
|
||||
const q = queue.data
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<div className="flex items-center justify-between">
|
||||
<h1 className="text-2xl font-semibold">Outbound queue</h1>
|
||||
<Button variant="outline" onClick={() => queue.refetch()} disabled={queue.isFetching}>
|
||||
Refresh
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
{queue.isPending && <p className="text-muted-foreground">Loading…</p>}
|
||||
{queue.isError && <p role="alert" className="text-destructive">{queue.error.message}</p>}
|
||||
|
||||
{q && (
|
||||
<>
|
||||
<div className="grid gap-4 sm:grid-cols-3">
|
||||
<Stat label="Pending" value={String(q.pending)} />
|
||||
<Stat label="Retrying" value={String(q.retrying)} />
|
||||
<Stat label="Oldest" value={q.oldest ? formatDate(q.oldest) : '—'} />
|
||||
</div>
|
||||
{q.items.length === 0 ? (
|
||||
<p className="text-muted-foreground">The queue is empty. Mail is sent as soon as it is accepted.</p>
|
||||
) : (
|
||||
<Table>
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead>Recipient</TableHead>
|
||||
<TableHead>Attempts</TableHead>
|
||||
<TableHead>Last error</TableHead>
|
||||
<TableHead>Next attempt</TableHead>
|
||||
<TableHead className="text-right">Actions</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{q.items.map((i) => (
|
||||
<TableRow key={i.id}>
|
||||
<TableCell className="font-medium">{i.recipient}</TableCell>
|
||||
<TableCell>{i.attempts}</TableCell>
|
||||
<TableCell>{i.last_error || '—'}</TableCell>
|
||||
<TableCell>{formatDate(i.next_attempt)}</TableCell>
|
||||
<TableCell className="text-right">
|
||||
<Button size="sm" variant="outline" onClick={() => setDiscarding(i)}>
|
||||
Discard
|
||||
</Button>
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
</Table>
|
||||
)}
|
||||
{q.pending > q.items.length && (
|
||||
<p className="text-sm text-muted-foreground">Showing {q.items.length} of {q.pending} deliveries.</p>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
<DiscardDialog item={discarding} onClose={() => setDiscarding(null)} />
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function Stat({ label, value }: { label: string; value: string }) {
|
||||
return (
|
||||
<Card size="sm">
|
||||
<CardContent>
|
||||
<div className="text-sm text-muted-foreground">{label}</div>
|
||||
<div className="text-xl font-semibold">{value}</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
)
|
||||
}
|
||||
|
||||
function DiscardDialog({ item, onClose }: { item: QueueItem | null; onClose: () => void }) {
|
||||
const queryClient = useQueryClient()
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const discard = useMutation({
|
||||
mutationFn: (i: QueueItem) => api.queue.discard(i.id),
|
||||
onSuccess: async () => {
|
||||
await queryClient.invalidateQueries({ queryKey: ['queue'] })
|
||||
toast.success('Delivery discarded')
|
||||
onClose()
|
||||
},
|
||||
onError: (e: Error) => setError(e.message),
|
||||
})
|
||||
|
||||
return (
|
||||
<AlertDialog
|
||||
open={item !== null}
|
||||
onOpenChange={(open) => {
|
||||
if (!open) {
|
||||
setError(null)
|
||||
onClose()
|
||||
}
|
||||
}}
|
||||
>
|
||||
<AlertDialogContent>
|
||||
<AlertDialogHeader>
|
||||
<AlertDialogTitle>Discard this delivery?</AlertDialogTitle>
|
||||
<AlertDialogDescription>
|
||||
The message for {item?.recipient} will not be sent. Other recipients are not affected.
|
||||
</AlertDialogDescription>
|
||||
</AlertDialogHeader>
|
||||
{error && (
|
||||
<p role="alert" className="text-sm text-destructive">
|
||||
{error}
|
||||
</p>
|
||||
)}
|
||||
<AlertDialogFooter>
|
||||
<AlertDialogCancel>Cancel</AlertDialogCancel>
|
||||
<Button variant="destructive" disabled={discard.isPending} onClick={() => item && discard.mutate(item)}>
|
||||
Discard
|
||||
</Button>
|
||||
</AlertDialogFooter>
|
||||
</AlertDialogContent>
|
||||
</AlertDialog>
|
||||
)
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
// An in-memory stand-in for the Parsimony API, used by component tests.
|
||||
import { vi } from 'vitest'
|
||||
import type { Held, List, Member, Message } from '@/lib/api'
|
||||
import type { Held, List, Member, Message, QueueItem } from '@/lib/api'
|
||||
|
||||
export interface FakeState {
|
||||
token: string
|
||||
|
|
@ -8,6 +8,9 @@ export interface FakeState {
|
|||
members: Record<number, Member[]>
|
||||
held: Record<number, Held[]>
|
||||
archive: Record<number, Message[]>
|
||||
queue: QueueItem[]
|
||||
/** Like the server, the queue listing is capped. */
|
||||
queueLimit: number
|
||||
nextId: number
|
||||
/** Every request seen, as "METHOD /path". */
|
||||
calls: string[]
|
||||
|
|
@ -16,7 +19,7 @@ export interface FakeState {
|
|||
}
|
||||
|
||||
export function newState(overrides: Partial<FakeState> = {}): FakeState {
|
||||
return { token: 'good', lists: [], members: {}, held: {}, archive: {}, nextId: 100, calls: [], fail: {}, ...overrides }
|
||||
return { token: 'good', lists: [], members: {}, held: {}, archive: {}, queue: [], queueLimit: 100, nextId: 100, calls: [], fail: {}, ...overrides }
|
||||
}
|
||||
|
||||
export function aList(over: Partial<List> = {}): List {
|
||||
|
|
@ -62,6 +65,18 @@ export function installFakeApi(state: FakeState) {
|
|||
const body = init.body ? JSON.parse(init.body as string) : {}
|
||||
let m: RegExpMatchArray | null
|
||||
|
||||
if (path === '/queue' && method === 'GET') {
|
||||
const items = [...state.queue].sort((a, b) => b.attempts - a.attempts).slice(0, state.queueLimit)
|
||||
const retrying = state.queue.filter((i) => i.attempts > 0).length
|
||||
const oldest = state.queue.map((i) => i.created_at).sort()[0] ?? null
|
||||
return json(200, { pending: state.queue.length, retrying, oldest, items })
|
||||
}
|
||||
if ((m = path.match(/^\/queue\/(\d+)$/)) && method === 'DELETE') {
|
||||
const id = Number(m[1])
|
||||
if (!state.queue.some((i) => i.id === id)) return err(404, 'not found')
|
||||
state.queue = state.queue.filter((i) => i.id !== id)
|
||||
return json(204)
|
||||
}
|
||||
if (path === '/lists' && method === 'GET') return json(200, state.lists)
|
||||
if (path === '/lists' && method === 'POST') {
|
||||
if (state.lists.some((l) => l.name === body.name && l.domain === body.domain)) return err(409, 'already exists')
|
||||
|
|
|
|||
|
|
@ -35,6 +35,15 @@ components:
|
|||
date: {type: string, format: date-time}
|
||||
size: {type: integer}
|
||||
body: {type: string, description: Only when reading a single message}
|
||||
QueueItem:
|
||||
type: object
|
||||
properties:
|
||||
id: {type: integer}
|
||||
recipient: {type: string}
|
||||
attempts: {type: integer, description: Failed attempts so far}
|
||||
last_error: {type: string}
|
||||
next_attempt: {type: string, format: date-time}
|
||||
created_at: {type: string, format: date-time}
|
||||
Member:
|
||||
type: object
|
||||
properties:
|
||||
|
|
@ -243,3 +252,26 @@ paths:
|
|||
responses:
|
||||
'204': {description: Deleted}
|
||||
'404': {$ref: '#/components/responses/Error'}
|
||||
/queue:
|
||||
get:
|
||||
summary: Outbound queue totals and up to 100 deliveries (failing ones first)
|
||||
responses:
|
||||
'200':
|
||||
description: OK
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
pending: {type: integer}
|
||||
retrying: {type: integer, description: Deliveries that failed at least once}
|
||||
oldest: {type: string, format: date-time, nullable: true}
|
||||
items: {type: array, items: {$ref: '#/components/schemas/QueueItem'}}
|
||||
/queue/{id}:
|
||||
parameters:
|
||||
- {name: id, in: path, required: true, schema: {type: integer}}
|
||||
delete:
|
||||
summary: Discard a queued delivery
|
||||
responses:
|
||||
'204': {description: Discarded}
|
||||
'404': {$ref: '#/components/responses/Error'}
|
||||
|
|
|
|||
|
|
@ -48,6 +48,8 @@ func New(s *store.Store, token string, rel Releaser) http.Handler {
|
|||
mux.Handle("GET /api/v1/lists/{id}/archive", srv.auth(srv.listArchive))
|
||||
mux.Handle("GET /api/v1/archive/{id}", srv.auth(srv.getArchived))
|
||||
mux.Handle("DELETE /api/v1/archive/{id}", srv.auth(srv.deleteArchived))
|
||||
mux.Handle("GET /api/v1/queue", srv.auth(srv.getQueue))
|
||||
mux.Handle("DELETE /api/v1/queue/{id}", srv.auth(srv.discardQueued))
|
||||
mux.Handle("GET /api/v1/lists/{id}/held", srv.auth(srv.listHeld))
|
||||
mux.Handle("POST /api/v1/held/{id}/approve", srv.auth(srv.approveHeld))
|
||||
mux.Handle("DELETE /api/v1/held/{id}", srv.auth(srv.discardHeld))
|
||||
|
|
@ -491,3 +493,53 @@ func (s *server) deleteArchived(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// Handlers: outbound queue.
|
||||
|
||||
const queueListLimit = 100
|
||||
|
||||
type queuedDTO struct {
|
||||
ID int64 `json:"id"`
|
||||
Recipient string `json:"recipient"`
|
||||
Attempts int `json:"attempts"`
|
||||
LastError string `json:"last_error"`
|
||||
NextAttempt time.Time `json:"next_attempt"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
func (s *server) getQueue(w http.ResponseWriter, _ *http.Request) {
|
||||
st, err := s.store.OutboxStats()
|
||||
if err != nil {
|
||||
storeError(w, err)
|
||||
return
|
||||
}
|
||||
items, err := s.store.ListOutbox(queueListLimit)
|
||||
if err != nil {
|
||||
storeError(w, err)
|
||||
return
|
||||
}
|
||||
out := make([]queuedDTO, 0, len(items))
|
||||
for _, it := range items {
|
||||
out = append(out, queuedDTO{ID: it.ID, Recipient: it.Rcpt, Attempts: it.Attempts, LastError: it.LastError,
|
||||
NextAttempt: it.NextAt, CreatedAt: it.CreatedAt})
|
||||
}
|
||||
var oldest *time.Time
|
||||
if !st.Oldest.IsZero() {
|
||||
oldest = &st.Oldest
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"pending": st.Pending, "retrying": st.Retrying, "oldest": oldest, "items": out,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *server) discardQueued(w http.ResponseWriter, r *http.Request) {
|
||||
id, ok := listID(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := s.store.CompleteOutbox(id); err != nil {
|
||||
storeError(w, err)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -435,3 +435,90 @@ func TestArchiveGetAndDelete(t *testing.T) {
|
|||
expect(t, do(t, h, "DELETE", "/api/v1/archive/1", "", true), 404)
|
||||
expect(t, do(t, h, "DELETE", "/api/v1/archive/2", "", false), 401)
|
||||
}
|
||||
|
||||
func queueServer(t *testing.T) (http.Handler, *store.Store) {
|
||||
t.Helper()
|
||||
s, _ := store.Open(":memory:")
|
||||
t.Cleanup(func() { s.Close() })
|
||||
return New(s, token, nil), s
|
||||
}
|
||||
|
||||
type queueJSON struct {
|
||||
Pending int `json:"pending"`
|
||||
Retrying int `json:"retrying"`
|
||||
Oldest *time.Time `json:"oldest"`
|
||||
Items []struct {
|
||||
ID int64 `json:"id"`
|
||||
Recipient string `json:"recipient"`
|
||||
Attempts int `json:"attempts"`
|
||||
LastError string `json:"last_error"`
|
||||
NextAttempt time.Time `json:"next_attempt"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
} `json:"items"`
|
||||
}
|
||||
|
||||
func TestQueueEmpty(t *testing.T) {
|
||||
h, _ := queueServer(t)
|
||||
rec := do(t, h, "GET", "/api/v1/queue", "", true)
|
||||
expect(t, rec, 200)
|
||||
var q queueJSON
|
||||
decode(t, rec, &q)
|
||||
if q.Pending != 0 || q.Retrying != 0 || q.Oldest != nil || q.Items == nil || len(q.Items) != 0 {
|
||||
t.Fatalf("empty queue: %+v (items must be [] not null)", q)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueueShowsPendingAndFailingDeliveries(t *testing.T) {
|
||||
h, s := queueServer(t)
|
||||
s.Enqueue([]byte("Subject: x\r\n\r\nx"), []store.NewOutbound{
|
||||
{From: "b@x.org", Rcpt: "a@x.com"}, {From: "b@x.org", Rcpt: "b@x.com"}, {From: "b@x.org", Rcpt: "c@x.com"},
|
||||
})
|
||||
due, _ := s.DueOutbox(10)
|
||||
s.RetryOutbox(due[1].ID, time.Now().Add(time.Hour), "connection refused")
|
||||
|
||||
rec := do(t, h, "GET", "/api/v1/queue", "", true)
|
||||
expect(t, rec, 200)
|
||||
var q queueJSON
|
||||
decode(t, rec, &q)
|
||||
if q.Pending != 3 || q.Retrying != 1 || q.Oldest == nil || len(q.Items) != 3 {
|
||||
t.Fatalf("queue: %+v", q)
|
||||
}
|
||||
if q.Items[0].Recipient != "b@x.com" || q.Items[0].Attempts != 1 || q.Items[0].LastError != "connection refused" {
|
||||
t.Fatalf("failing deliveries come first: %+v", q.Items[0])
|
||||
}
|
||||
if strings.Contains(rec.Body.String(), "Subject") {
|
||||
t.Fatal("message contents must not be exposed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueueListIsCapped(t *testing.T) {
|
||||
h, s := queueServer(t)
|
||||
var items []store.NewOutbound
|
||||
for i := 0; i < 120; i++ {
|
||||
items = append(items, store.NewOutbound{From: "b@x.org", Rcpt: fmt.Sprintf("u%d@x.com", i)})
|
||||
}
|
||||
s.Enqueue([]byte("x"), items)
|
||||
var q queueJSON
|
||||
decode(t, do(t, h, "GET", "/api/v1/queue", "", true), &q)
|
||||
if q.Pending != 120 || len(q.Items) != 100 {
|
||||
t.Fatalf("pending=%d items=%d", q.Pending, len(q.Items))
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueueDiscard(t *testing.T) {
|
||||
h, s := queueServer(t)
|
||||
s.Enqueue([]byte("x"), []store.NewOutbound{{From: "b@x.org", Rcpt: "a@x.com"}, {From: "b@x.org", Rcpt: "b@x.com"}})
|
||||
due, _ := s.DueOutbox(10)
|
||||
expect(t, do(t, h, "DELETE", fmt.Sprintf("/api/v1/queue/%d", due[0].ID), "", true), 204)
|
||||
expect(t, do(t, h, "DELETE", fmt.Sprintf("/api/v1/queue/%d", due[0].ID), "", true), 404)
|
||||
expect(t, do(t, h, "DELETE", "/api/v1/queue/abc", "", true), 404)
|
||||
if st, _ := s.OutboxStats(); st.Pending != 1 {
|
||||
t.Fatalf("pending = %d", st.Pending)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueueRequiresAuth(t *testing.T) {
|
||||
h, _ := queueServer(t)
|
||||
expect(t, do(t, h, "GET", "/api/v1/queue", "", false), 401)
|
||||
expect(t, do(t, h, "DELETE", "/api/v1/queue/1", "", false), 401)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,11 +19,15 @@ import (
|
|||
"codeberg.org/squidd/parsimony/internal/config"
|
||||
"codeberg.org/squidd/parsimony/internal/ingest"
|
||||
"codeberg.org/squidd/parsimony/internal/outbound"
|
||||
"codeberg.org/squidd/parsimony/internal/queue"
|
||||
"codeberg.org/squidd/parsimony/internal/router"
|
||||
"codeberg.org/squidd/parsimony/internal/store"
|
||||
"codeberg.org/squidd/parsimony/internal/web"
|
||||
)
|
||||
|
||||
// nowFunc is the clock of the store (queue schedules, token expiry). Tests move it.
|
||||
var nowFunc = time.Now
|
||||
|
||||
// App is a running daemon.
|
||||
type App struct {
|
||||
// LMTPAddr and APIAddr are the addresses actually bound.
|
||||
|
|
@ -38,16 +42,14 @@ type App struct {
|
|||
web *http.Server
|
||||
pub *http.Server
|
||||
wg sync.WaitGroup
|
||||
stop context.CancelFunc // stops the queue worker
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
// releaser lets the API approve held messages through the router.
|
||||
type releaser struct {
|
||||
rt *router.Router
|
||||
out outbound.Sender
|
||||
}
|
||||
type releaser struct{ rt *router.Router }
|
||||
|
||||
func (r releaser) Release(ctx context.Context, id int64) error { return r.rt.Release(ctx, r.out, id) }
|
||||
func (r releaser) Release(_ context.Context, id int64) error { return r.rt.Release(id) }
|
||||
|
||||
// Start opens the database and begins serving. The REST API runs only when
|
||||
// apiToken is not empty.
|
||||
|
|
@ -56,12 +58,18 @@ func Start(cfg config.Config, apiToken string) (*App, error) {
|
|||
if err != nil {
|
||||
return nil, fmt.Errorf("store: %w", err)
|
||||
}
|
||||
st.Now = nowFunc
|
||||
a := &App{store: st}
|
||||
|
||||
rt := router.New(st)
|
||||
rt.BaseURL = cfg.BaseURL
|
||||
relay := &outbound.SMTPRelay{Addr: cfg.Relay.Addr}
|
||||
a.lmtp = ingest.NewServer(rt, relay)
|
||||
worker := &queue.Worker{
|
||||
Store: st,
|
||||
Sender: &outbound.SMTPRelay{Addr: cfg.Relay.Addr},
|
||||
Concurrency: cfg.Relay.Concurrency,
|
||||
}
|
||||
rt.OnEnqueue = worker.Wake
|
||||
a.lmtp = ingest.NewServer(rt)
|
||||
|
||||
if cfg.LMTP.Network == "unix" {
|
||||
os.Remove(cfg.LMTP.Addr)
|
||||
|
|
@ -86,7 +94,7 @@ func Start(cfg config.Config, apiToken string) (*App, error) {
|
|||
a.lns = append(a.lns, apiLn)
|
||||
// The admin UI and the API share a listener, so the UI needs no CORS.
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("/api/", api.New(st, apiToken, releaser{rt, relay}))
|
||||
mux.Handle("/api/", api.New(st, apiToken, releaser{rt}))
|
||||
mux.Handle("/", adminui.Embedded())
|
||||
a.web = &http.Server{Handler: mux, ReadHeaderTimeout: 10 * time.Second}
|
||||
}
|
||||
|
|
@ -106,6 +114,13 @@ func Start(cfg config.Config, apiToken string) (*App, error) {
|
|||
a.pub = &http.Server{Handler: web.New(st, rt), ReadHeaderTimeout: 10 * time.Second}
|
||||
}
|
||||
|
||||
var workerCtx context.Context
|
||||
workerCtx, a.stop = context.WithCancel(context.Background())
|
||||
a.wg.Add(1)
|
||||
go func() {
|
||||
defer a.wg.Done()
|
||||
worker.Run(workerCtx)
|
||||
}()
|
||||
a.wg.Add(1)
|
||||
go func() {
|
||||
defer a.wg.Done()
|
||||
|
|
@ -147,6 +162,7 @@ func (a *App) Close() {
|
|||
if a.pub != nil {
|
||||
a.pub.Close()
|
||||
}
|
||||
a.stop() // in-flight sends are interrupted and stay queued
|
||||
a.wg.Wait()
|
||||
a.store.Close()
|
||||
})
|
||||
|
|
|
|||
|
|
@ -6,12 +6,14 @@ import (
|
|||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/emersion/go-smtp"
|
||||
|
||||
"codeberg.org/squidd/parsimony/internal/config"
|
||||
"codeberg.org/squidd/parsimony/internal/store"
|
||||
)
|
||||
|
||||
// sink is a fake SMTP relay that records what it receives.
|
||||
|
|
@ -58,6 +60,18 @@ func (s *sink) count() int {
|
|||
return len(s.msgs)
|
||||
}
|
||||
|
||||
// waitCount waits until the relay holds n messages.
|
||||
func (s *sink) waitCount(t *testing.T, n int) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for s.count() < n {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("relay has %d messages, want %d", s.count(), n)
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *sink) all() []sunk {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
|
@ -168,16 +182,15 @@ func TestEndToEnd(t *testing.T) {
|
|||
if err := lmtpSend(t, a.LMTPAddr, "alice@example.com", "dev@example.org", msg); err != nil {
|
||||
t.Fatalf("lmtp: %v", err)
|
||||
}
|
||||
if relay.count() != 2 {
|
||||
t.Fatalf("relay got %d messages, want 2", relay.count())
|
||||
}
|
||||
got := relay.msgs[0]
|
||||
relay.waitCount(t, 2)
|
||||
got := relay.all()[0]
|
||||
if !strings.Contains(got.data, "List-Id: Dev <dev.example.org>") || !strings.HasPrefix(got.from, "dev+bounce-") {
|
||||
t.Fatalf("unexpected relayed mail: %+v", got)
|
||||
}
|
||||
if err := lmtpSend(t, a.LMTPAddr, "mallory@evil.com", "dev@example.org", msg); err == nil {
|
||||
t.Fatal("non-member must be rejected")
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond) // give a wrongly queued message time to show up
|
||||
if relay.count() != 2 {
|
||||
t.Fatal("rejected mail must not be relayed")
|
||||
}
|
||||
|
|
@ -275,6 +288,7 @@ func TestArchiveAndUnsubscribeEndToEnd(t *testing.T) {
|
|||
}
|
||||
|
||||
// The relayed copy carries a working one-click link for that recipient.
|
||||
relay.waitCount(t, 2)
|
||||
var link string
|
||||
for _, m := range relay.all() {
|
||||
if len(m.to) == 1 && m.to[0] == "bob@example.com" {
|
||||
|
|
@ -300,6 +314,8 @@ func TestArchiveAndUnsubscribeEndToEnd(t *testing.T) {
|
|||
"From: Alice <alice@example.com>\r\nSubject: Second\r\nMessage-ID: <q2@example.com>\r\n\r\nhi\r\n"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
relay.waitCount(t, before+1)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if got := relay.count() - before; got != 1 {
|
||||
t.Fatalf("after bob left only alice should get a copy, got %d", got)
|
||||
}
|
||||
|
|
@ -386,3 +402,122 @@ func TestAdminUIAndAPIShareTheListener(t *testing.T) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
// deadAddr returns an address nothing listens on.
|
||||
func deadAddr(t *testing.T) string {
|
||||
t.Helper()
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
addr := ln.Addr().String()
|
||||
ln.Close()
|
||||
return addr
|
||||
}
|
||||
|
||||
func TestQueuedMailSurvivesARestartAndIsRetriedWhenTheRelayReturns(t *testing.T) {
|
||||
var skew atomic.Int64
|
||||
nowFunc = func() time.Time { return time.Now().Add(time.Duration(skew.Load())) }
|
||||
t.Cleanup(func() { nowFunc = time.Now })
|
||||
|
||||
// 1. The relay is down: the post is still accepted and stays queued.
|
||||
cfg := testConfig(t, deadAddr(t))
|
||||
a, err := Start(cfg, "tok")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org","title":"Dev"}`, "tok")
|
||||
for _, m := range []string{"alice@example.com", "bob@example.com"} {
|
||||
call(t, a, "POST", "/api/v1/lists/1/members", `{"address":"`+m+`"}`, "tok")
|
||||
}
|
||||
msg := "From: Alice <alice@example.com>\r\nSubject: queued\r\nMessage-ID: <q@x>\r\n\r\nbody\r\n"
|
||||
if err := lmtpSend(t, a.LMTPAddr, "alice@example.com", "dev@example.org", msg); err != nil {
|
||||
t.Fatalf("a post must be accepted while the relay is down: %v", err)
|
||||
}
|
||||
peek, err := store.Open(cfg.DB)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer peek.Close()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
st, _ := peek.OutboxStats()
|
||||
if st.Pending == 2 && st.Retrying == 2 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("queue after the first attempts: %+v", st)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
// 2. Restart with a working relay, after the backoff has passed.
|
||||
a.Close()
|
||||
relayAddr, relay := startSink(t)
|
||||
cfg.Relay.Addr = relayAddr
|
||||
skew.Store(int64(10 * time.Minute))
|
||||
b, err := Start(cfg, "tok")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { b.Close() })
|
||||
relay.waitCount(t, 2)
|
||||
got := relay.all()
|
||||
if !strings.Contains(got[0].data, "List-Id: Dev <dev.example.org>") || !strings.Contains(got[0].data, "Subject: queued") {
|
||||
t.Fatalf("delivered mail: %q", got[0].data)
|
||||
}
|
||||
deadline = time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
st, _ := peek.OutboxStats()
|
||||
if st.Pending == 0 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("delivered mail must leave the queue: %+v", st)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloseInterruptsASlowSendWithoutLosingIt(t *testing.T) {
|
||||
// A relay that accepts the connection and never answers.
|
||||
ln, _ := net.Listen("tcp", "127.0.0.1:0")
|
||||
t.Cleanup(func() { ln.Close() })
|
||||
go func() {
|
||||
for {
|
||||
c, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer c.Close()
|
||||
}
|
||||
}()
|
||||
cfg := testConfig(t, ln.Addr().String())
|
||||
a, err := Start(cfg, "tok")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org"}`, "tok")
|
||||
call(t, a, "POST", "/api/v1/lists/1/members", `{"address":"alice@example.com"}`, "tok")
|
||||
msg := "From: Alice <alice@example.com>\r\nSubject: slow\r\nMessage-ID: <s@x>\r\n\r\nbody\r\n"
|
||||
if err := lmtpSend(t, a.LMTPAddr, "alice@example.com", "dev@example.org", msg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond) // the worker is now stuck talking to the silent relay
|
||||
|
||||
closed := make(chan struct{})
|
||||
go func() { a.Close(); close(closed) }()
|
||||
select {
|
||||
case <-closed:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Close must not wait for a hung relay")
|
||||
}
|
||||
peek, err := store.Open(cfg.DB)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer peek.Close()
|
||||
if st, _ := peek.OutboxStats(); st.Pending != 1 {
|
||||
t.Fatalf("the interrupted delivery must stay queued: %+v", st)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,7 +24,8 @@ type Config struct {
|
|||
Listen string `yaml:"listen"` // public archive and unsubscribe pages; empty disables them
|
||||
} `yaml:"web"`
|
||||
Relay struct {
|
||||
Addr string `yaml:"addr"` // host:port of the outgoing SMTP relay
|
||||
Addr string `yaml:"addr"` // host:port of the outgoing SMTP relay
|
||||
Concurrency int `yaml:"concurrency"` // simultaneous deliveries
|
||||
} `yaml:"relay"`
|
||||
}
|
||||
|
||||
|
|
@ -35,6 +36,7 @@ func Load(path string) (Config, error) {
|
|||
c.LMTP.Network = "tcp"
|
||||
c.LMTP.Addr = "127.0.0.1:2424"
|
||||
c.Relay.Addr = "127.0.0.1:25"
|
||||
c.Relay.Concurrency = 4
|
||||
c.API.Listen = "127.0.0.1:8080"
|
||||
c.Web.Listen = "127.0.0.1:8081"
|
||||
|
||||
|
|
|
|||
|
|
@ -20,17 +20,17 @@ func TestLoadDefaults(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.DB != "parsimony.db" || c.LMTP.Network != "tcp" || c.LMTP.Addr != "127.0.0.1:2424" || c.Relay.Addr != "127.0.0.1:25" || c.API.Listen != "127.0.0.1:8080" || c.Web.Listen != "127.0.0.1:8081" {
|
||||
if c.DB != "parsimony.db" || c.LMTP.Network != "tcp" || c.LMTP.Addr != "127.0.0.1:2424" || c.Relay.Addr != "127.0.0.1:25" || c.Relay.Concurrency != 4 || c.API.Listen != "127.0.0.1:8080" || c.Web.Listen != "127.0.0.1:8081" {
|
||||
t.Fatalf("defaults: %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadValues(t *testing.T) {
|
||||
c, err := Load(write(t, "db: /var/lib/p.db\nbase_url: https://l.example.org\nlmtp:\n network: unix\n addr: /run/p.sock\nrelay:\n addr: mx:587\napi:\n listen: 0.0.0.0:9000\nweb:\n listen: 0.0.0.0:9001\n"))
|
||||
c, err := Load(write(t, "db: /var/lib/p.db\nbase_url: https://l.example.org\nlmtp:\n network: unix\n addr: /run/p.sock\nrelay:\n addr: mx:587\n concurrency: 9\napi:\n listen: 0.0.0.0:9000\nweb:\n listen: 0.0.0.0:9001\n"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.DB != "/var/lib/p.db" || c.BaseURL != "https://l.example.org" || c.LMTP.Network != "unix" || c.LMTP.Addr != "/run/p.sock" || c.Relay.Addr != "mx:587" || c.API.Listen != "0.0.0.0:9000" || c.Web.Listen != "0.0.0.0:9001" {
|
||||
if c.DB != "/var/lib/p.db" || c.BaseURL != "https://l.example.org" || c.LMTP.Network != "unix" || c.LMTP.Addr != "/run/p.sock" || c.Relay.Addr != "mx:587" || c.Relay.Concurrency != 9 || c.API.Listen != "0.0.0.0:9000" || c.Web.Listen != "0.0.0.0:9001" {
|
||||
t.Fatalf("values: %+v", c)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
package ingest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
|
|
@ -10,16 +9,16 @@ import (
|
|||
|
||||
"github.com/emersion/go-smtp"
|
||||
|
||||
"codeberg.org/squidd/parsimony/internal/outbound"
|
||||
"codeberg.org/squidd/parsimony/internal/router"
|
||||
)
|
||||
|
||||
const maxMessageBytes = 25 << 20
|
||||
|
||||
// NewServer builds an LMTP server that routes incoming mail and relays it.
|
||||
// NewServer builds an LMTP server that routes incoming mail into the outbound queue.
|
||||
// A message is accepted only once it is durably queued.
|
||||
// Set Network/Addr on the result (or call Serve) to start it.
|
||||
func NewServer(r *router.Router, out outbound.Sender) *smtp.Server {
|
||||
srv := smtp.NewServer(&backend{router: r, out: out})
|
||||
func NewServer(r *router.Router) *smtp.Server {
|
||||
srv := smtp.NewServer(&backend{router: r})
|
||||
srv.LMTP = true
|
||||
srv.Domain = "localhost"
|
||||
srv.MaxMessageBytes = maxMessageBytes
|
||||
|
|
@ -30,7 +29,6 @@ func NewServer(r *router.Router, out outbound.Sender) *smtp.Server {
|
|||
|
||||
type backend struct {
|
||||
router *router.Router
|
||||
out outbound.Sender
|
||||
}
|
||||
|
||||
func (b *backend) NewSession(*smtp.Conn) (smtp.Session, error) {
|
||||
|
|
@ -67,25 +65,22 @@ func (s *session) LMTPData(r io.Reader, status smtp.StatusCollector) error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
for _, rcpt := range s.rcpts {
|
||||
status.SetStatus(rcpt, s.deliver(ctx, rcpt, data))
|
||||
status.SetStatus(rcpt, s.deliver(rcpt, data))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *session) deliver(ctx context.Context, rcpt string, data []byte) error {
|
||||
func (s *session) deliver(rcpt string, data []byte) error {
|
||||
rt := s.b.router
|
||||
if handled, err := rt.Command(ctx, s.b.out, s.from, rcpt, data); handled {
|
||||
if handled, err := rt.Command(s.from, rcpt, data); handled {
|
||||
return s.status(rcpt, err)
|
||||
}
|
||||
res, err := rt.Route(s.from, rcpt, data)
|
||||
if err != nil {
|
||||
return s.status(rcpt, err)
|
||||
}
|
||||
return s.status(rcpt, rt.Send(ctx, s.b.out, res))
|
||||
return s.status(rcpt, rt.Deliver(res))
|
||||
}
|
||||
|
||||
// status turns a router result into an LMTP reply.
|
||||
|
|
|
|||
|
|
@ -6,17 +6,26 @@ import (
|
|||
"net"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/emersion/go-smtp"
|
||||
|
||||
"codeberg.org/squidd/parsimony/internal/outbound"
|
||||
"codeberg.org/squidd/parsimony/internal/queue"
|
||||
"codeberg.org/squidd/parsimony/internal/router"
|
||||
"codeberg.org/squidd/parsimony/internal/store"
|
||||
)
|
||||
|
||||
const msg = "From: Alice <alice@example.com>\r\nSubject: hi\r\n\r\nbody\r\n"
|
||||
|
||||
func setup(t *testing.T) (addr string, relay *outbound.Fake) {
|
||||
// stack is the LMTP server with a real queue worker in front of a fake relay.
|
||||
type stack struct {
|
||||
addr string
|
||||
relay *outbound.Fake
|
||||
store *store.Store
|
||||
}
|
||||
|
||||
func newStack(t *testing.T, sender outbound.Sender) *stack {
|
||||
t.Helper()
|
||||
s, err := store.Open(":memory:")
|
||||
if err != nil {
|
||||
|
|
@ -27,15 +36,52 @@ func setup(t *testing.T) (addr string, relay *outbound.Fake) {
|
|||
s.AddMember(l.ID, "alice@example.com", "Alice")
|
||||
s.AddMember(l.ID, "bob@example.com", "Bob")
|
||||
|
||||
relay = &outbound.Fake{}
|
||||
srv := NewServer(router.New(s), relay)
|
||||
relay := &outbound.Fake{}
|
||||
if sender == nil {
|
||||
sender = relay
|
||||
}
|
||||
rt := router.New(s)
|
||||
w := &queue.Worker{Store: s, Sender: sender, Interval: 10 * time.Millisecond}
|
||||
rt.OnEnqueue = w.Wake
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { w.Run(ctx); close(done) }()
|
||||
t.Cleanup(func() { cancel(); <-done })
|
||||
|
||||
srv := NewServer(rt)
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go srv.Serve(ln)
|
||||
t.Cleanup(func() { srv.Close() })
|
||||
return ln.Addr().String(), relay
|
||||
return &stack{addr: ln.Addr().String(), relay: relay, store: s}
|
||||
}
|
||||
|
||||
// sent waits until the relay has received n messages and returns them.
|
||||
func (st *stack) sent(t *testing.T, n int) []outbound.Sent {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for {
|
||||
got := st.relay.Messages()
|
||||
if len(got) >= n {
|
||||
return got
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("relay got %d messages, want %d", len(got), n)
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// queued is how many deliveries still wait in the outbox.
|
||||
func (st *stack) queued(t *testing.T) int {
|
||||
t.Helper()
|
||||
stats, err := st.store.OutboxStats()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return stats.Pending
|
||||
}
|
||||
|
||||
// deliver sends msg over LMTP and returns the per-recipient errors.
|
||||
|
|
@ -80,41 +126,45 @@ func deliver(t *testing.T, addr, from string, rcpts []string) map[string]error {
|
|||
}
|
||||
|
||||
func TestDeliverFansOut(t *testing.T) {
|
||||
addr, relay := setup(t)
|
||||
res := deliver(t, addr, "alice@example.com", []string{"dev@example.org"})
|
||||
st := newStack(t, nil)
|
||||
res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org"})
|
||||
if res["dev@example.org"] != nil {
|
||||
t.Fatalf("unexpected error: %v", res)
|
||||
}
|
||||
if len(relay.Sent) != 2 {
|
||||
t.Fatalf("want one send per member, got %d", len(relay.Sent))
|
||||
got := st.sent(t, 2)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("want one send per member, got %d", len(got))
|
||||
}
|
||||
s := relay.Sent[0]
|
||||
if len(s.To) != 1 {
|
||||
t.Errorf("recipients = %v", s.To)
|
||||
m := got[0]
|
||||
if len(m.To) != 1 {
|
||||
t.Errorf("recipients = %v", m.To)
|
||||
}
|
||||
if !strings.HasPrefix(s.From, "dev+bounce-") {
|
||||
t.Errorf("envelope from = %q", s.From)
|
||||
if !strings.HasPrefix(m.From, "dev+bounce-") {
|
||||
t.Errorf("envelope from = %q", m.From)
|
||||
}
|
||||
if !strings.Contains(string(s.Data), "List-Id: Dev <dev.example.org>") {
|
||||
t.Errorf("missing List-Id: %q", s.Data)
|
||||
if !strings.Contains(string(m.Data), "List-Id: Dev <dev.example.org>") {
|
||||
t.Errorf("missing List-Id: %q", m.Data)
|
||||
}
|
||||
if !strings.Contains(string(m.Data), "List-Unsubscribe: <mailto:dev+leave@example.org>") {
|
||||
t.Errorf("missing per-recipient unsubscribe header: %q", m.Data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNonMemberRejectedPermanently(t *testing.T) {
|
||||
addr, relay := setup(t)
|
||||
res := deliver(t, addr, "mallory@evil.com", []string{"dev@example.org"})
|
||||
st := newStack(t, nil)
|
||||
res := deliver(t, st.addr, "mallory@evil.com", []string{"dev@example.org"})
|
||||
var se *smtp.SMTPError
|
||||
if !errors.As(res["dev@example.org"], &se) || se.Code != 550 {
|
||||
t.Fatalf("want 550, got %v", res["dev@example.org"])
|
||||
}
|
||||
if len(relay.Sent) != 0 {
|
||||
t.Fatal("nothing must be relayed")
|
||||
if st.queued(t) != 0 {
|
||||
t.Fatal("nothing may be queued")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnknownListRejected(t *testing.T) {
|
||||
addr, _ := setup(t)
|
||||
res := deliver(t, addr, "alice@example.com", []string{"nope@example.org"})
|
||||
st := newStack(t, nil)
|
||||
res := deliver(t, st.addr, "alice@example.com", []string{"nope@example.org"})
|
||||
var se *smtp.SMTPError
|
||||
if !errors.As(res["nope@example.org"], &se) || se.Code != 550 {
|
||||
t.Fatalf("want 550, got %v", res["nope@example.org"])
|
||||
|
|
@ -122,79 +172,83 @@ func TestUnknownListRejected(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestMixedRecipients(t *testing.T) {
|
||||
addr, relay := setup(t)
|
||||
res := deliver(t, addr, "alice@example.com", []string{"dev@example.org", "nope@example.org"})
|
||||
st := newStack(t, nil)
|
||||
res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org", "nope@example.org"})
|
||||
if res["dev@example.org"] != nil || res["nope@example.org"] == nil {
|
||||
t.Fatalf("results: %v", res)
|
||||
}
|
||||
if len(relay.Sent) != 2 {
|
||||
t.Fatalf("sent = %d", len(relay.Sent))
|
||||
st.sent(t, 2)
|
||||
}
|
||||
|
||||
// down is a relay that refuses everything.
|
||||
type down struct{}
|
||||
|
||||
func (down) Send(context.Context, string, []string, []byte) error { return errors.New("relay down") }
|
||||
|
||||
func TestRelayDownStillAcceptsAndKeepsTheMailQueued(t *testing.T) {
|
||||
st := newStack(t, down{})
|
||||
res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org"})
|
||||
if res["dev@example.org"] != nil {
|
||||
t.Fatalf("a queued post is accepted even when the relay is down: %v", res)
|
||||
}
|
||||
// The worker tries and schedules retries; nothing is lost.
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for {
|
||||
stats, _ := st.store.OutboxStats()
|
||||
if stats.Pending == 2 && stats.Retrying == 2 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("queue: %+v", stats)
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
type failing struct{}
|
||||
|
||||
func (failing) Send(_ context.Context, _ string, _ []string, _ []byte) error {
|
||||
return errors.New("relay down")
|
||||
}
|
||||
|
||||
func TestRelayFailureIsTemporary(t *testing.T) {
|
||||
s, _ := store.Open(":memory:")
|
||||
t.Cleanup(func() { s.Close() })
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
s.AddMember(l.ID, "alice@example.com", "Alice")
|
||||
srv := NewServer(router.New(s), failing{})
|
||||
ln, _ := net.Listen("tcp", "127.0.0.1:0")
|
||||
go srv.Serve(ln)
|
||||
t.Cleanup(func() { srv.Close() })
|
||||
|
||||
res := deliver(t, ln.Addr().String(), "alice@example.com", []string{"dev@example.org"})
|
||||
func TestQueueFailureIsTemporary(t *testing.T) {
|
||||
st := newStack(t, nil)
|
||||
st.store.Close() // the database goes away under the server
|
||||
res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org"})
|
||||
var se *smtp.SMTPError
|
||||
if !errors.As(res["dev@example.org"], &se) || se.Code != 451 {
|
||||
t.Fatalf("want 451, got %v", res["dev@example.org"])
|
||||
t.Fatalf("want 451 so the MTA retries, got %v", res["dev@example.org"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestJoinCommandOverLMTP(t *testing.T) {
|
||||
addr, relay := setup(t)
|
||||
res := deliver(t, addr, "carol@x.com", []string{"dev+join@example.org"})
|
||||
st := newStack(t, nil)
|
||||
res := deliver(t, st.addr, "carol@x.com", []string{"dev+join@example.org"})
|
||||
if res["dev+join@example.org"] != nil {
|
||||
t.Fatalf("join: %v", res)
|
||||
}
|
||||
if len(relay.Sent) != 1 || relay.Sent[0].To[0] != "carol@x.com" {
|
||||
t.Fatalf("confirmation mail: %+v", relay.Sent)
|
||||
got := st.sent(t, 1)
|
||||
if len(got[0].To) != 1 || got[0].To[0] != "carol@x.com" {
|
||||
t.Fatalf("confirmation mail: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBadTokenRejected(t *testing.T) {
|
||||
addr, _ := setup(t)
|
||||
res := deliver(t, addr, "carol@x.com", []string{"dev+confirm-deadbeef@example.org"})
|
||||
st := newStack(t, nil)
|
||||
res := deliver(t, st.addr, "carol@x.com", []string{"dev+confirm-deadbeef@example.org"})
|
||||
var se *smtp.SMTPError
|
||||
if !errors.As(res["dev+confirm-deadbeef@example.org"], &se) || se.Code != 550 {
|
||||
t.Fatalf("want 550, got %v", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModeratedPostIsAcceptedButNotRelayed(t *testing.T) {
|
||||
s, _ := store.Open(":memory:")
|
||||
t.Cleanup(func() { s.Close() })
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
s.AddMember(l.ID, "alice@example.com", "Alice")
|
||||
s.SetPolicies(l.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate)
|
||||
relay := &outbound.Fake{}
|
||||
srv := NewServer(router.New(s), relay)
|
||||
ln, _ := net.Listen("tcp", "127.0.0.1:0")
|
||||
go srv.Serve(ln)
|
||||
t.Cleanup(func() { srv.Close() })
|
||||
func TestModeratedPostIsAcceptedButNotQueued(t *testing.T) {
|
||||
st := newStack(t, nil)
|
||||
l, _ := st.store.FindList("dev@example.org")
|
||||
st.store.SetPolicies(l.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate)
|
||||
|
||||
res := deliver(t, ln.Addr().String(), "stranger@x.com", []string{"dev@example.org"})
|
||||
res := deliver(t, st.addr, "stranger@x.com", []string{"dev@example.org"})
|
||||
if res["dev@example.org"] != nil {
|
||||
t.Fatalf("held mail must be accepted: %v", res)
|
||||
}
|
||||
if len(relay.Sent) != 0 {
|
||||
t.Fatal("nothing may be relayed")
|
||||
if st.queued(t) != 0 {
|
||||
t.Fatal("nothing may be queued for delivery")
|
||||
}
|
||||
if h, _ := s.ListHeld(l.ID); len(h) != 1 {
|
||||
if h, _ := st.store.ListHeld(l.ID); len(h) != 1 {
|
||||
t.Fatalf("held = %d", len(h))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ package outbound
|
|||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"sync"
|
||||
|
|
@ -36,18 +37,41 @@ func (r *SMTPRelay) Send(ctx context.Context, from string, to []string, data []b
|
|||
for start := 0; start < len(to); start += size {
|
||||
end := min(start+size, len(to))
|
||||
if err := r.sendBatch(ctx, from, to[start:end], data); err != nil {
|
||||
if len(to) <= size {
|
||||
return err // a single batch: the range adds nothing
|
||||
}
|
||||
return fmt.Errorf("batch %d-%d: %w", start, end, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *SMTPRelay) sendBatch(ctx context.Context, from string, to []string, data []byte) error {
|
||||
func (r *SMTPRelay) sendBatch(ctx context.Context, from string, to []string, data []byte) (err error) {
|
||||
d := net.Dialer{Timeout: 10 * time.Second}
|
||||
conn, err := d.DialContext(ctx, "tcp", r.Addr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// A relay that stops answering must not hold us past the context: apply its
|
||||
// deadline to the connection and close the connection if it is cancelled.
|
||||
if dl, ok := ctx.Deadline(); ok {
|
||||
conn.SetDeadline(dl)
|
||||
}
|
||||
stop := make(chan struct{})
|
||||
defer close(stop)
|
||||
go func() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
conn.Close()
|
||||
case <-stop:
|
||||
}
|
||||
}()
|
||||
defer func() {
|
||||
if err != nil && ctx.Err() != nil {
|
||||
err = ctx.Err() // report the cause, not "use of closed connection"
|
||||
}
|
||||
}()
|
||||
|
||||
c := smtp.NewClient(conn)
|
||||
defer c.Close()
|
||||
if err := c.Hello("localhost"); err != nil {
|
||||
|
|
@ -72,6 +96,13 @@ type Fake struct {
|
|||
Sent []Sent
|
||||
}
|
||||
|
||||
// Messages returns a copy of what was recorded; safe while sends are running.
|
||||
func (f *Fake) Messages() []Sent {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]Sent(nil), f.Sent...)
|
||||
}
|
||||
|
||||
// Send records the message.
|
||||
func (f *Fake) Send(_ context.Context, from string, to []string, data []byte) error {
|
||||
f.mu.Lock()
|
||||
|
|
@ -79,3 +110,36 @@ func (f *Fake) Send(_ context.Context, from string, to []string, data []byte) er
|
|||
f.Sent = append(f.Sent, Sent{from, append([]string(nil), to...), append([]byte(nil), data...)})
|
||||
return nil
|
||||
}
|
||||
|
||||
// PermanentError marks a failure that retrying cannot fix. SMTP 5xx replies are
|
||||
// permanent without being wrapped in it; use it for other sources (and in tests).
|
||||
type PermanentError struct {
|
||||
Err error
|
||||
BadMailbox bool // the destination mailbox is invalid
|
||||
}
|
||||
|
||||
func (e *PermanentError) Error() string { return e.Err.Error() }
|
||||
func (e *PermanentError) Unwrap() error { return e.Err }
|
||||
|
||||
// IsPermanent reports whether err is a permanent failure: an SMTP 5xx reply or a PermanentError.
|
||||
func IsPermanent(err error) bool {
|
||||
var pe *PermanentError
|
||||
if errors.As(err, &pe) {
|
||||
return true
|
||||
}
|
||||
var se *smtp.SMTPError
|
||||
return errors.As(err, &se) && se.Code >= 500 && se.Code <= 599
|
||||
}
|
||||
|
||||
// IsBadMailbox reports whether err says the destination mailbox is invalid (SMTP
|
||||
// enhanced status 5.1.x). Other permanent errors, such as "relay access denied",
|
||||
// say nothing about the recipient.
|
||||
func IsBadMailbox(err error) bool {
|
||||
var pe *PermanentError
|
||||
if errors.As(err, &pe) {
|
||||
return pe.BadMailbox
|
||||
}
|
||||
var se *smtp.SMTPError
|
||||
return errors.As(err, &se) && se.Code >= 500 && se.Code <= 599 &&
|
||||
se.EnhancedCode[0] == 5 && se.EnhancedCode[1] == 1
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,11 +2,14 @@ package outbound
|
|||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/emersion/go-smtp"
|
||||
)
|
||||
|
|
@ -128,3 +131,151 @@ func TestSMTPRelayDefaultBatchSize(t *testing.T) {
|
|||
t.Fatalf("default batch size must be %d, got %d transactions", DefaultBatchSize, len(cap.msgs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyErrors(t *testing.T) {
|
||||
smtpErr := func(code int, enh smtp.EnhancedCode) error {
|
||||
return &smtp.SMTPError{Code: code, EnhancedCode: enh, Message: "x"}
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
err error
|
||||
perm, mailbox bool
|
||||
}{
|
||||
{"nil", nil, false, false},
|
||||
{"plain", errors.New("connection refused"), false, false},
|
||||
{"451", smtpErr(451, smtp.EnhancedCode{4, 3, 0}), false, false},
|
||||
{"421", smtpErr(421, smtp.NoEnhancedCode), false, false},
|
||||
{"550 unknown mailbox", smtpErr(550, smtp.EnhancedCode{5, 1, 1}), true, true},
|
||||
{"553 bad address syntax", smtpErr(553, smtp.EnhancedCode{5, 1, 3}), true, true},
|
||||
{"554 relay denied", smtpErr(554, smtp.EnhancedCode{5, 7, 1}), true, false},
|
||||
{"550 no enhanced code", smtpErr(550, smtp.NoEnhancedCode), true, false},
|
||||
{"wrapped", fmt.Errorf("batch 0-1: %w", smtpErr(550, smtp.EnhancedCode{5, 1, 1})), true, true},
|
||||
{"explicit permanent", &PermanentError{Err: errors.New("no")}, true, false},
|
||||
{"explicit bad mailbox", &PermanentError{Err: errors.New("no"), BadMailbox: true}, true, true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := IsPermanent(c.err); got != c.perm {
|
||||
t.Errorf("%s: IsPermanent = %v", c.name, got)
|
||||
}
|
||||
if got := IsBadMailbox(c.err); got != c.mailbox {
|
||||
t.Errorf("%s: IsBadMailbox = %v", c.name, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type rejecting struct{ err error }
|
||||
|
||||
func (r rejecting) NewSession(*smtp.Conn) (smtp.Session, error) { return &rejectSession{r.err}, nil }
|
||||
|
||||
type rejectSession struct{ err error }
|
||||
|
||||
func (s *rejectSession) Reset() {}
|
||||
func (s *rejectSession) Logout() error { return nil }
|
||||
func (s *rejectSession) Mail(string, *smtp.MailOptions) error { return nil }
|
||||
func (s *rejectSession) Rcpt(string, *smtp.RcptOptions) error { return s.err }
|
||||
func (s *rejectSession) Data(io.Reader) error { return nil }
|
||||
|
||||
func TestRelayRejectionsAreClassifiedFromTheWire(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
err error
|
||||
perm, mailbox bool
|
||||
}{
|
||||
"unknown user": {&smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"}, true, true},
|
||||
"relay denied": {&smtp.SMTPError{Code: 554, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "relay access denied"}, true, false},
|
||||
"try later": {&smtp.SMTPError{Code: 451, EnhancedCode: smtp.EnhancedCode{4, 3, 0}, Message: "try later"}, false, false},
|
||||
} {
|
||||
srv := smtp.NewServer(rejecting{tc.err})
|
||||
ln, _ := net.Listen("tcp", "127.0.0.1:0")
|
||||
go srv.Serve(ln)
|
||||
r := &SMTPRelay{Addr: ln.Addr().String()}
|
||||
err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, []byte("Subject: x\r\n\r\nx\r\n"))
|
||||
srv.Close()
|
||||
if err == nil {
|
||||
t.Fatalf("%s: want an error", name)
|
||||
}
|
||||
if IsPermanent(err) != tc.perm || IsBadMailbox(err) != tc.mailbox {
|
||||
t.Errorf("%s: permanent=%v mailbox=%v for %v", name, IsPermanent(err), IsBadMailbox(err), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// silent accepts connections and never says a word.
|
||||
func silent(t *testing.T) string {
|
||||
t.Helper()
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var conns []net.Conn
|
||||
var mu sync.Mutex
|
||||
go func() {
|
||||
for {
|
||||
c, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
conns = append(conns, c)
|
||||
mu.Unlock()
|
||||
}
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
ln.Close()
|
||||
mu.Lock()
|
||||
for _, c := range conns {
|
||||
c.Close()
|
||||
}
|
||||
mu.Unlock()
|
||||
})
|
||||
return ln.Addr().String()
|
||||
}
|
||||
|
||||
func TestSendStopsWhenTheContextIsCancelled(t *testing.T) {
|
||||
r := &SMTPRelay{Addr: silent(t)}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
errc := make(chan error, 1)
|
||||
go func() { errc <- r.Send(ctx, "b@x.org", []string{"a@x.com"}, []byte("x")) }()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
cancel()
|
||||
select {
|
||||
case err := <-errc:
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("want context.Canceled, got %v", err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("Send ignored the cancelled context while talking to a silent relay")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSendHonoursTheContextDeadline(t *testing.T) {
|
||||
r := &SMTPRelay{Addr: silent(t)}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond)
|
||||
defer cancel()
|
||||
start := time.Now()
|
||||
err := r.Send(ctx, "b@x.org", []string{"a@x.com"}, []byte("x"))
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("want context.DeadlineExceeded, got %v", err)
|
||||
}
|
||||
if took := time.Since(start); took > 2*time.Second {
|
||||
t.Fatalf("took %v", took)
|
||||
}
|
||||
if IsPermanent(err) {
|
||||
t.Fatalf("a timeout is temporary: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSingleBatchErrorsAreNotDecoratedWithBatchNumbers(t *testing.T) {
|
||||
r := &SMTPRelay{Addr: "127.0.0.1:1"}
|
||||
err := r.Send(context.Background(), "b@x.org", []string{"a@x.com"}, []byte("x"))
|
||||
if err == nil || strings.HasPrefix(err.Error(), "batch") {
|
||||
t.Fatalf("one recipient, one batch: the error should be the plain cause, got %v", err)
|
||||
}
|
||||
many := make([]string, 120)
|
||||
for i := range many {
|
||||
many[i] = fmt.Sprintf("u%d@x.com", i)
|
||||
}
|
||||
err = (&SMTPRelay{Addr: "127.0.0.1:1", BatchSize: 50}).Send(context.Background(), "b@x.org", many, []byte("x"))
|
||||
if err == nil || !strings.HasPrefix(err.Error(), "batch 0-50:") {
|
||||
t.Fatalf("with several batches the failing range is useful: %v", err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
211
internal/queue/queue.go
Normal file
211
internal/queue/queue.go
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
// Package queue delivers the outbox to the SMTP relay in the background.
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"codeberg.org/squidd/parsimony/internal/outbound"
|
||||
"codeberg.org/squidd/parsimony/internal/store"
|
||||
)
|
||||
|
||||
// MaxAge is how long a failing delivery is retried before it is dropped.
|
||||
const MaxAge = 72 * time.Hour
|
||||
|
||||
// Defaults for the zero values of Worker fields.
|
||||
const (
|
||||
defaultConcurrency = 4
|
||||
defaultBatch = 50
|
||||
defaultTimeout = time.Minute
|
||||
defaultInterval = 5 * time.Second
|
||||
)
|
||||
|
||||
var backoff = []time.Duration{time.Minute, 5 * time.Minute, 15 * time.Minute, time.Hour, 3 * time.Hour, 6 * time.Hour}
|
||||
|
||||
// Backoff returns how long to wait after the n-th consecutive failure (n >= 1).
|
||||
func Backoff(failures int) time.Duration {
|
||||
if failures < 1 {
|
||||
failures = 1
|
||||
}
|
||||
if failures > len(backoff) {
|
||||
failures = len(backoff)
|
||||
}
|
||||
return backoff[failures-1]
|
||||
}
|
||||
|
||||
// Stats counts what one run did.
|
||||
type Stats struct {
|
||||
Sent int // delivered to the relay
|
||||
Retried int // failed temporarily, scheduled again
|
||||
Dropped int // removed: permanent failure or too old
|
||||
Bounced int // dropped deliveries that counted as a hard bounce
|
||||
}
|
||||
|
||||
// Worker sends queued deliveries. Zero values of the optional fields mean defaults.
|
||||
type Worker struct {
|
||||
Store *store.Store
|
||||
Sender outbound.Sender
|
||||
Concurrency int // simultaneous sends
|
||||
Batch int // deliveries taken per run
|
||||
Timeout time.Duration // per send
|
||||
Interval time.Duration // idle polling, to pick up retries that have become due
|
||||
|
||||
wakeOnce sync.Once
|
||||
wake chan struct{}
|
||||
runMu sync.Mutex
|
||||
}
|
||||
|
||||
func (w *Worker) wakeCh() chan struct{} {
|
||||
w.wakeOnce.Do(func() { w.wake = make(chan struct{}, 1) })
|
||||
return w.wake
|
||||
}
|
||||
|
||||
// Wake asks Run to look at the queue now. It never blocks.
|
||||
func (w *Worker) Wake() {
|
||||
select {
|
||||
case w.wakeCh() <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// Run delivers until ctx is cancelled.
|
||||
func (w *Worker) Run(ctx context.Context) {
|
||||
interval := w.Interval
|
||||
if interval <= 0 {
|
||||
interval = defaultInterval
|
||||
}
|
||||
timer := time.NewTimer(0)
|
||||
defer timer.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-w.wakeCh():
|
||||
case <-timer.C:
|
||||
}
|
||||
// Keep going while there is work, so a big post drains without waiting for the timer.
|
||||
for ctx.Err() == nil {
|
||||
st, err := w.RunOnce(ctx)
|
||||
if err != nil {
|
||||
log.Printf("queue: %v", err)
|
||||
break
|
||||
}
|
||||
if st == (Stats{}) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if !timer.Stop() {
|
||||
select {
|
||||
case <-timer.C:
|
||||
default:
|
||||
}
|
||||
}
|
||||
timer.Reset(interval)
|
||||
}
|
||||
}
|
||||
|
||||
// RunOnce attempts every delivery that is due, up to Batch, with bounded concurrency.
|
||||
func (w *Worker) RunOnce(ctx context.Context) (Stats, error) {
|
||||
w.runMu.Lock()
|
||||
defer w.runMu.Unlock()
|
||||
|
||||
batch := w.Batch
|
||||
if batch <= 0 {
|
||||
batch = defaultBatch
|
||||
}
|
||||
conc := w.Concurrency
|
||||
if conc <= 0 {
|
||||
conc = defaultConcurrency
|
||||
}
|
||||
due, err := w.Store.DueOutbox(batch)
|
||||
if err != nil {
|
||||
return Stats{}, err
|
||||
}
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
stats Stats
|
||||
wg sync.WaitGroup
|
||||
sem = make(chan struct{}, conc)
|
||||
)
|
||||
for _, it := range due {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
sem <- struct{}{}
|
||||
wg.Add(1)
|
||||
go func(it store.OutboxItem) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
o := w.deliver(ctx, it)
|
||||
mu.Lock()
|
||||
stats.Sent += o.Sent
|
||||
stats.Retried += o.Retried
|
||||
stats.Dropped += o.Dropped
|
||||
stats.Bounced += o.Bounced
|
||||
mu.Unlock()
|
||||
}(it)
|
||||
}
|
||||
wg.Wait()
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func (w *Worker) deliver(ctx context.Context, it store.OutboxItem) Stats {
|
||||
data, err := w.Store.OutboxData(it.MsgID)
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
w.complete(it)
|
||||
return Stats{Dropped: 1}
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("queue: read message %d: %v", it.MsgID, err)
|
||||
return Stats{}
|
||||
}
|
||||
msg := make([]byte, 0, len(it.Prefix)+len(data))
|
||||
msg = append(append(msg, it.Prefix...), data...)
|
||||
|
||||
timeout := w.Timeout
|
||||
if timeout <= 0 {
|
||||
timeout = defaultTimeout
|
||||
}
|
||||
sctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
err = w.Sender.Send(sctx, it.From, []string{it.Rcpt}, msg)
|
||||
cancel()
|
||||
|
||||
switch {
|
||||
case err == nil:
|
||||
w.complete(it)
|
||||
return Stats{Sent: 1}
|
||||
case ctx.Err() != nil:
|
||||
return Stats{} // shutting down: the delivery stays queued, unchanged
|
||||
case outbound.IsPermanent(err):
|
||||
out := Stats{Dropped: 1}
|
||||
if outbound.IsBadMailbox(err) && it.MemberID != 0 {
|
||||
if _, e := w.Store.RecordHardBounce(it.MemberID, store.BounceThreshold); e == nil {
|
||||
out.Bounced = 1
|
||||
}
|
||||
}
|
||||
log.Printf("queue: drop %s: %v", it.Rcpt, err)
|
||||
w.complete(it)
|
||||
return out
|
||||
}
|
||||
now := w.Store.Now()
|
||||
if now.Sub(it.CreatedAt) > MaxAge {
|
||||
log.Printf("queue: give up on %s after %s: %v", it.Rcpt, MaxAge, err)
|
||||
w.complete(it)
|
||||
return Stats{Dropped: 1}
|
||||
}
|
||||
if e := w.Store.RetryOutbox(it.ID, now.Add(Backoff(it.Attempts+1)), err.Error()); e != nil {
|
||||
log.Printf("queue: reschedule %d: %v", it.ID, e)
|
||||
return Stats{}
|
||||
}
|
||||
return Stats{Retried: 1}
|
||||
}
|
||||
|
||||
func (w *Worker) complete(it store.OutboxItem) {
|
||||
if err := w.Store.CompleteOutbox(it.ID); err != nil && !errors.Is(err, store.ErrNotFound) {
|
||||
log.Printf("queue: remove %d: %v", it.ID, err)
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue