diff --git a/README.md b/README.md index 8245173..4feb7be 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ A modern, single-binary mailing list manager: an evolution of Mailman. send to `list+join@domain` or `list+leave@domain`, then reply to the confirmation. A reply works once, a request expires after 72 hours, and a confirmation that could not be applied (closed list, database error) can be replied to again. A member disabled by bounces can rejoin the same way. +- Newsletters: a `publishers` post policy, sent-post statistics, a relay rate limit and a public subscribe form. - Moderated lists: posts wait in a queue that the API can approve or discard. - VERP bounces: every recipient gets a signed return address; 3 hard bounces within 30 days disable a member. - Archive with full-text search (SQLite FTS5) and threading. Each list has an @@ -56,6 +57,30 @@ The admin UI and the public archive, with demo data (`docs/screenshots/`). ![A public archive page with the thread of a message](docs/screenshots/public-archive.png) +## Newsletters + +A newsletter is a list whose post policy is `publishers`: only members flagged as publishers may post, and everybody else just receives. Everything else (double opt-in, one-click unsubscribe, DKIM, archive, bounces, the queue) works as for a discussion list. + +```sh +parsimony list set news@example.org --post publishers +parsimony member add news@example.org alice@example.com --publisher +parsimony member set news@example.org alice@example.com --publisher=false # demote +``` + +In the admin UI, pick "Publishers only (newsletter)" in the list's Settings tab and use the "Can publish" checkbox or the "Make publisher" action on the Members tab. The API uses `PATCH /api/v1/lists/{id}/members/{address}` with `{"publisher": true}`. `member import` accepts `email,name,publisher` lines. `member ls` shows each member's role. + +A publisher's post passes the same identity check as any post: the `From` header must name the SMTP sender, or the post is held. A post from anyone else, member or not, is refused with `550 5.7.1`; it is never held, archived or queued. A publisher disabled by bounces can still post. + +**Sent posts.** Each post queued to a list is recorded with its subject, time and number of recipients, and the queue counts how its deliveries end: **sent** (accepted by the relay), **failed** (refused, or still failing after 3 days), **dropped** (the mailbox does not exist, or the delivery was discarded) and **pending**. A count changes in the same transaction as the delivery. See them with `parsimony posts ls news@example.org`, `GET /api/v1/lists/{id}/posts`, or the Sent tab of a list, which refreshes every 5 seconds while something is pending. Posts are kept for 400 days. Parsimony does not track opens or clicks. + +**Priority.** Newsletter deliveries go out after everything else that is due, so a big newsletter never delays discussion mail or confirmations. + +**Rate limiting.** Relays such as Amazon SES throttle sending. Set `relay.rate_per_second` to the most sends the daemon may start in one second (default `0`, unlimited). One pacer sits in front of every send attempt, retries included, and spaces them evenly. Set it a little under your relay's limit (12 for a limit of 14). Each recipient is one message, so a post to N members takes about N divided by the rate in seconds. + +**Subscribe form.** A list whose archive is public and whose join policy is open gets a signup page at `/subscribe/{address}` on the public web listener, linked from its archive page. It sends the usual double opt-in message. The page that follows is always the same, whether or not the address is a member or the list qualifies for mail, so the form cannot be used to find out who is subscribed. A hidden field catches bots; each IP (the TCP peer, `X-Forwarded-For` is ignored) may post 10 times per hour, then gets `429`; behind a reverse proxy every visitor shares its address, so limit at the proxy too. The body is capped at 4 KiB. + +Not included: scheduled sends and open or click tracking. + ## Run ``` diff --git a/admin/src/App.tsx b/admin/src/App.tsx index e832cba..91339ba 100644 --- a/admin/src/App.tsx +++ b/admin/src/App.tsx @@ -7,6 +7,7 @@ import { NotFoundPage } from '@/pages/NotFoundPage' import { ArchiveTab } from '@/tabs/ArchiveTab' import { HeldTab } from '@/tabs/HeldTab' import { MembersTab } from '@/tabs/MembersTab' +import { SentTab } from '@/tabs/SentTab' import { SettingsTab } from '@/tabs/SettingsTab' import { QueuePage } from '@/pages/QueuePage' import { Toaster } from '@/components/ui/sonner' @@ -25,6 +26,7 @@ export function App() { } /> } /> } /> + } /> } /> diff --git a/admin/src/lib/api.ts b/admin/src/lib/api.ts index 0f25352..cec8c9d 100644 --- a/admin/src/lib/api.ts +++ b/admin/src/lib/api.ts @@ -1,6 +1,6 @@ // Typed client for the Parsimony admin REST API (/api/v1). -export type PostPolicy = 'members' | 'moderated' +export type PostPolicy = 'members' | 'moderated' | 'publishers' export type JoinPolicy = 'open' | 'closed' export type ArchivePolicy = 'public' | 'private' | 'off' @@ -20,6 +20,8 @@ export interface Member { display_name: string disabled: boolean bounce_count: number + /** May post when the list's post policy is 'publishers'. */ + publisher: boolean } export interface Held { @@ -79,6 +81,22 @@ export interface Queue { items: QueueItem[] } +/** A post queued to a list, with how far its delivery got. */ +export interface Post { + id: number + message_id: string + subject: string + created_at: string + /** Deliveries queued: the members when it was posted. */ + recipients: number + sent: number + failed: number + /** The mailbox does not exist, or the delivery was discarded. */ + dropped: number + /** Still in the queue. */ + pending: number +} + export class ApiError extends Error { readonly status: number constructor(status: number, message: string) { @@ -154,8 +172,11 @@ export const api = { }, members: { list: (listId: number) => request('GET', `/lists/${listId}/members`), - add: (listId: number, b: { address: string; display_name?: string }) => + /** Without publisher an existing member keeps its flag. */ + add: (listId: number, b: { address: string; display_name?: string; publisher?: boolean }) => request('POST', `/lists/${listId}/members`, b), + setPublisher: (listId: number, address: string, publisher: boolean) => + request('PATCH', `/lists/${listId}/members/${enc(address)}`, { publisher }), remove: (listId: number, address: string) => request('DELETE', `/lists/${listId}/members/${enc(address)}`), }, @@ -164,6 +185,9 @@ export const api = { approve: (id: number) => request('POST', `/held/${id}/approve`), discard: (id: number) => request('DELETE', `/held/${id}`), }, + posts: { + list: (listId: number) => request('GET', `/lists/${listId}/posts`), + }, queue: { get: () => request('GET', '/queue'), discard: (id: number) => request('DELETE', `/queue/${id}`), diff --git a/admin/src/pages/ListPage.tsx b/admin/src/pages/ListPage.tsx index 4d65251..77ff633 100644 --- a/admin/src/pages/ListPage.tsx +++ b/admin/src/pages/ListPage.tsx @@ -4,7 +4,7 @@ import { Badge } from '@/components/ui/badge' import { Tabs, TabsList, TabsTrigger } from '@/components/ui/tabs' import { api } from '@/lib/api' -const tabs = ['settings', 'members', 'held', 'archive'] as const +const tabs = ['settings', 'members', 'held', 'archive', 'sent'] as const export function ListPage() { const params = useParams() @@ -42,6 +42,7 @@ export function ListPage() { {heldCount > 0 && {heldCount}} Archive + Sent diff --git a/admin/src/tabs/MembersTab.test.tsx b/admin/src/tabs/MembersTab.test.tsx index f69ca62..f5f42b0 100644 --- a/admin/src/tabs/MembersTab.test.tsx +++ b/admin/src/tabs/MembersTab.test.tsx @@ -9,6 +9,7 @@ const member = (address: string, over: Partial = {}): Member => ({ display_name: '', disabled: false, bounce_count: 0, + publisher: false, ...over, }) @@ -105,6 +106,72 @@ describe('members tab', () => { expect(screen.queryByRole('button', { name: /re-enable/i })).not.toBeInTheDocument() }) + describe('publishers', () => { + it('shows a Publisher badge on publishers only', async () => { + await setup([member('alice@example.com', { publisher: true }), member('bob@example.com')]) + const alice = (await screen.findByText('alice@example.com')).closest('tr')! + expect(within(alice).getByText('Publisher')).toBeInTheDocument() + const bob = screen.getByText('bob@example.com').closest('tr')! + expect(within(bob).queryByText('Publisher')).not.toBeInTheDocument() + }) + + it('adds a publisher with the Can publish checkbox, then clears it', async () => { + const { user, state } = await setup() + await user.type(screen.getByLabelText(/email address/i), 'dave@example.com') + await user.click(screen.getByRole('checkbox', { name: /can publish/i })) + await user.click(screen.getByRole('button', { name: /add member/i })) + const row = (await screen.findByText('dave@example.com')).closest('tr')! + expect(await within(row).findByText('Publisher')).toBeInTheDocument() + expect(state.members[1][0].publisher).toBe(true) + expect(screen.getByRole('checkbox', { name: /can publish/i })).not.toBeChecked() + }) + + it('adding without the checkbox leaves an existing publisher alone', async () => { + const { user, state } = await setup([member('dave@example.com', { publisher: true })]) + await screen.findByText('dave@example.com') + expect(screen.getByRole('checkbox', { name: /can publish/i })).not.toBeChecked() + await user.type(screen.getByLabelText(/email address/i), 'dave@example.com') + await user.type(screen.getByLabelText(/display name/i), 'Dave') + await user.click(screen.getByRole('button', { name: /add member/i })) + await waitFor(() => expect(state.members[1][0].display_name).toBe('Dave')) + expect(state.members[1][0].publisher).toBe(true) + }) + + it('re-enabling keeps the publisher flag', async () => { + const { user, state } = await setup([member('bob@example.com', { disabled: true, publisher: true })]) + const row = (await screen.findByText('bob@example.com')).closest('tr')! + await user.click(within(row).getByRole('button', { name: /re-enable/i })) + await waitFor(() => expect(state.members[1][0].disabled).toBe(false)) + expect(state.members[1][0].publisher).toBe(true) + }) + + it('makes a member a publisher and back', async () => { + const { user, state } = await setup([member('a+b@example.com')]) + let row = (await screen.findByText('a+b@example.com')).closest('tr')! + await user.click(within(row).getByRole('button', { name: /make publisher/i })) + await waitFor(() => expect(state.members[1][0].publisher).toBe(true)) + expect(state.calls).toContain('PATCH /lists/1/members/a%2Bb%40example.com') + row = screen.getByText('a+b@example.com').closest('tr')! + expect(await within(row).findByText('Publisher')).toBeInTheDocument() + expect(await screen.findByText(/a\+b@example.com can now publish/i)).toBeInTheDocument() + + await user.click(within(row).getByRole('button', { name: /make subscriber/i })) + await waitFor(() => expect(state.members[1][0].publisher).toBe(false)) + row = screen.getByText('a+b@example.com').closest('tr')! + await waitFor(() => expect(within(row).queryByText('Publisher')).not.toBeInTheDocument()) + expect(within(row).getByRole('button', { name: /make publisher/i })).toBeInTheDocument() + }) + + it('shows the server error of the row action', async () => { + const { user, state } = await setup([member('alice@example.com')]) + const row = (await screen.findByText('alice@example.com')).closest('tr')! + state.fail['PATCH /lists/1/members/alice%40example.com'] = { status: 500, error: 'locked' } + await user.click(within(row).getByRole('button', { name: /make publisher/i })) + expect(await screen.findByText('locked')).toBeInTheDocument() + expect(state.members[1][0].publisher).toBe(false) + }) + }) + describe('removing', () => { async function open() { const ctx = await setup([member('alice@example.com'), member('a+b@example.com')]) diff --git a/admin/src/tabs/MembersTab.tsx b/admin/src/tabs/MembersTab.tsx index 6a669ef..7a1e8bf 100644 --- a/admin/src/tabs/MembersTab.tsx +++ b/admin/src/tabs/MembersTab.tsx @@ -36,6 +36,15 @@ export function MembersTab() { onError: (e: Error) => toast.error(e.message), }) + const setPublisher = useMutation({ + mutationFn: (m: Member) => api.members.setPublisher(list.id, m.address, !m.publisher), + onSuccess: async (_, m) => { + await queryClient.invalidateQueries({ queryKey: key }) + toast.success(m.publisher ? `${m.address} is now a subscriber` : `${m.address} can now publish`) + }, + onError: (e: Error) => toast.error(e.message), + }) + const all = members.data ?? [] const q = filter.trim().toLowerCase() const shown = q ? all.filter((m) => m.address.includes(q) || m.display_name.toLowerCase().includes(q)) : all @@ -76,6 +85,7 @@ export function MembersTab() { {m.address} {m.display_name} + {m.publisher && Publisher} {m.disabled ? Disabled : Active} {m.bounce_count > 0 && ( @@ -89,6 +99,9 @@ export function MembersTab() { Re-enable )} + @@ -109,15 +122,19 @@ function AddMemberForm({ list }: { list: List }) { const queryClient = useQueryClient() const [address, setAddress] = useState('') const [name, setName] = useState('') + const [publisher, setPublisher] = useState(false) const [error, setError] = useState(null) const add = useMutation({ - mutationFn: () => api.members.add(list.id, { address: address.trim(), display_name: name.trim() }), + // Unchecked sends no flag, so adding an existing publisher again does not demote it. + mutationFn: () => + api.members.add(list.id, { address: address.trim(), display_name: name.trim(), ...(publisher ? { publisher: true } : {}) }), onSuccess: async (m) => { await queryClient.invalidateQueries({ queryKey: ['members', list.id] }) toast.success(`Added ${m.address}`) setAddress('') setName('') + setPublisher(false) setError(null) }, onError: (e: Error) => setError(e.message), @@ -144,6 +161,16 @@ function AddMemberForm({ list }: { list: List }) { setName(e.target.value)} autoComplete="off" /> +
+ setPublisher(e.target.checked)} + /> + +
diff --git a/admin/src/tabs/SentTab.test.tsx b/admin/src/tabs/SentTab.test.tsx new file mode 100644 index 0000000..35fc02f --- /dev/null +++ b/admin/src/tabs/SentTab.test.tsx @@ -0,0 +1,113 @@ +import { act, screen, within } from '@testing-library/react' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { Post } from '@/lib/api' +import { aList, installFakeApi, newState } from '@/test/fakeApi' +import { renderApp } from '@/test/render' + +const post = (id: number, over: Partial = {}): Post => ({ + id, + message_id: `<${id}@example.org>`, + subject: `Issue ${id}`, + created_at: '2030-01-02T03:04:05Z', + recipients: 4, + sent: 4, + failed: 0, + dropped: 0, + pending: 0, + ...over, +}) + +async function setup(posts: Post[]) { + const state = installFakeApi(newState({ lists: [aList({ id: 1 })], posts: { 1: posts } })) + const ctx = renderApp('/lists/1/sent', { token: 'good' }) + await screen.findByRole('heading', { name: /sent posts/i, level: 2 }) + return { ...ctx, state } +} + +const postCalls = (calls: string[]) => calls.filter((c) => c === 'GET /lists/1/posts').length + +afterEach(() => vi.useRealTimers()) + +describe('sent tab', () => { + it('is a tab of the list page', async () => { + installFakeApi(newState({ lists: [aList({ id: 1 })], posts: { 1: [post(1)] } })) + const { user } = renderApp('/lists/1', { token: 'good' }) + await user.click(await screen.findByRole('tab', { name: /^sent$/i })) + expect(await screen.findByRole('heading', { name: /sent posts/i, level: 2 })).toBeInTheDocument() + expect(await screen.findByText('Issue 1')).toBeInTheDocument() + expect(screen.getByRole('tab', { name: /^sent$/i })).toHaveAttribute('aria-selected', 'true') + }) + + it('says when nothing was sent yet', async () => { + await setup([]) + expect(await screen.findByText(/nothing has been sent to this list yet/i)).toBeInTheDocument() + expect(screen.queryByRole('table')).not.toBeInTheDocument() + }) + + it('shows each post with its counts and progress', async () => { + await setup([ + post(2, { subject: '', recipients: 10, sent: 6, failed: 1, dropped: 1, pending: 2 }), + post(1), + ]) + const row = (await screen.findByText('(no subject)')).closest('tr')! + expect(within(row).getByText('2030-01-02 03:04 UTC')).toBeInTheDocument() + const bar = within(row).getByRole('progressbar') + expect(bar).toHaveAttribute('aria-valuenow', '6') + expect(bar).toHaveAttribute('aria-valuemax', '10') + expect(within(row).getByText('6 of 10')).toBeInTheDocument() + const cells = within(row).getAllByRole('cell').map((c) => c.textContent) + expect(cells.slice(3)).toEqual(['10', '2', '1', '1']) + const headers = screen.getAllByRole('columnheader').map((h) => h.textContent) + expect(headers).toEqual(['Subject', 'Date', 'Sent', 'Recipients', 'Pending', 'Failed', 'Dropped']) + const done = screen.getByText('Issue 1').closest('tr')! + expect(within(done).getByRole('progressbar')).toHaveAttribute('aria-valuenow', '4') + const rows = screen.getAllByRole('row').slice(1).map((r) => r.textContent ?? '') + expect(rows[0]).toContain('(no subject)') // the order of the server: newest first + }) + + it('shows a post nobody received without a progress bar', async () => { + await setup([post(1, { recipients: 0, sent: 0 })]) + const row = (await screen.findByText('Issue 1')).closest('tr')! + expect(within(row).queryByRole('progressbar')).not.toBeInTheDocument() + expect(within(row).getByText(/no recipients/i)).toBeInTheDocument() + }) + + it('shows subjects as text', async () => { + await setup([post(1, { subject: '' })]) + expect(await screen.findByText('')).toBeInTheDocument() + expect(document.querySelector('img[src="x"]')).toBeNull() + }) + + it('refreshes every few seconds while a post is being sent', async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }) + const { state } = await setup([post(1, { sent: 1, pending: 3 })]) + await screen.findByText('1 of 4') + state.posts[1] = [post(1)] + await act(async () => { + await vi.advanceTimersByTimeAsync(6000) + }) + expect(await screen.findByText('4 of 4')).toBeInTheDocument() + // Everything is sent: no more polling. + const before = postCalls(state.calls) + await act(async () => { + await vi.advanceTimersByTimeAsync(12000) + }) + expect(postCalls(state.calls)).toBe(before) + }) + + it('does not refresh when nothing is pending', async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }) + const { state } = await setup([post(1)]) + await screen.findByText('4 of 4') + await act(async () => { + await vi.advanceTimersByTimeAsync(12000) + }) + expect(postCalls(state.calls)).toBe(1) + }) + + it('shows a loading error', async () => { + installFakeApi(newState({ lists: [aList({ id: 1 })], fail: { 'GET /lists/1/posts': { status: 500, error: 'database is locked' } } })) + renderApp('/lists/1/sent', { token: 'good' }) + expect(await screen.findByRole('alert')).toHaveTextContent('database is locked') + }) +}) diff --git a/admin/src/tabs/SentTab.tsx b/admin/src/tabs/SentTab.tsx new file mode 100644 index 0000000..9426b0e --- /dev/null +++ b/admin/src/tabs/SentTab.tsx @@ -0,0 +1,81 @@ +import { useQuery } from '@tanstack/react-query' +import { useOutletContext } from 'react-router' +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table' +import { api, type List, type Post } from '@/lib/api' +import { formatDate } from '@/lib/format' + +/** How often the tab reloads by itself while a post is still being sent. */ +export const REFRESH_MS = 5000 + +export function SentTab() { + const list = useOutletContext() + const posts = useQuery({ + queryKey: ['posts', list.id], + queryFn: () => api.posts.list(list.id), + refetchInterval: (q) => (q.state.data?.some((p) => p.pending > 0) ? REFRESH_MS : false), + }) + const items = posts.data ?? [] + + return ( +
+

Sent posts

+ {posts.isPending &&

Loading…

} + {posts.isError &&

{posts.error.message}

} + {posts.data && items.length === 0 && ( +

Nothing has been sent to this list yet.

+ )} + {items.length > 0 && ( + + + + Subject + Date + Sent + Recipients + Pending + Failed + Dropped + + + + {items.map((p) => ( + + {p.subject || '(no subject)'} + {formatDate(p.created_at)} + + + + {p.recipients} + {p.pending} + {p.failed} + {p.dropped} + + ))} + +
+ )} +
+ ) +} + +function Progress({ post }: { post: Post }) { + if (post.recipients === 0) return No recipients + const pct = Math.round((100 * post.sent) / post.recipients) + return ( +
+
+
+
+ + {post.sent} of {post.recipients} + +
+ ) +} diff --git a/admin/src/tabs/SettingsTab.test.tsx b/admin/src/tabs/SettingsTab.test.tsx new file mode 100644 index 0000000..4d5c5c5 --- /dev/null +++ b/admin/src/tabs/SettingsTab.test.tsx @@ -0,0 +1,24 @@ +import { screen, waitFor } from '@testing-library/react' +import { describe, expect, it } from 'vitest' +import { aList, installFakeApi, newState } from '@/test/fakeApi' +import { renderApp } from '@/test/render' + +describe('settings tab: newsletter', () => { + it('shows the publishers policy', async () => { + installFakeApi(newState({ lists: [aList({ id: 1, post_policy: 'publishers' })] })) + renderApp('/lists/1', { token: 'good' }) + await screen.findByLabelText(/^title$/i) + expect(screen.getByRole('combobox', { name: /posting/i })).toHaveTextContent('Publishers only (newsletter)') + }) + + it('turns a list into a newsletter', async () => { + const state = installFakeApi(newState({ lists: [aList({ id: 1 })] })) + const { user } = renderApp('/lists/1', { token: 'good' }) + await screen.findByLabelText(/^title$/i) + await user.click(screen.getByRole('combobox', { name: /posting/i })) + await user.click(await screen.findByRole('option', { name: 'Publishers only (newsletter)' })) + await user.click(screen.getByRole('button', { name: /save changes/i })) + await waitFor(() => expect(state.lists[0].post_policy).toBe('publishers')) + expect(state.lists[0].join_policy).toBe('open') + }) +}) diff --git a/admin/src/tabs/SettingsTab.tsx b/admin/src/tabs/SettingsTab.tsx index 4502b12..51ec6b5 100644 --- a/admin/src/tabs/SettingsTab.tsx +++ b/admin/src/tabs/SettingsTab.tsx @@ -21,6 +21,7 @@ import { api, type ArchivePolicy, type JoinPolicy, type List, type PostPolicy } const postOptions: Option[] = [ { value: 'members', label: 'Members only' }, { value: 'moderated', label: 'Moderated (hold every post)' }, + { value: 'publishers', label: 'Publishers only (newsletter)' }, ] const joinOptions: Option[] = [ { value: 'open', label: 'Open (confirmed by email)' }, diff --git a/admin/src/test/fakeApi.ts b/admin/src/test/fakeApi.ts index c8bbfaf..43a192a 100644 --- a/admin/src/test/fakeApi.ts +++ b/admin/src/test/fakeApi.ts @@ -1,6 +1,6 @@ // An in-memory stand-in for the Parsimony API, used by component tests. import { vi } from 'vitest' -import type { Held, List, Member, Message, QueueItem } from '@/lib/api' +import type { Held, List, Member, Message, Post, QueueItem } from '@/lib/api' export interface FakeState { token: string @@ -9,6 +9,7 @@ export interface FakeState { held: Record archive: Record queue: QueueItem[] + posts: Record /** Like the server, the queue listing is capped. */ queueLimit: number /** Content types seen on archive imports. */ @@ -23,7 +24,7 @@ export interface FakeState { } export function newState(overrides: Partial = {}): FakeState { - return { token: 'good', lists: [], members: {}, held: {}, archive: {}, queue: [], queueLimit: 100, importTypes: [], nextId: 100, calls: [], fail: {}, ...overrides } + return { token: 'good', lists: [], members: {}, held: {}, archive: {}, queue: [], posts: {}, queueLimit: 100, importTypes: [], nextId: 100, calls: [], fail: {}, ...overrides } } export function aList(over: Partial = {}): List { @@ -112,11 +113,20 @@ export function installFakeApi(state: FakeState) { if (method === 'GET') return json(200, ms) if (method === 'POST') { if (!/^[^@\s]+@[^@\s]+$/.test(body.address ?? '')) return err(400, 'a valid email address is required') - const mem: Member = { address: body.address.toLowerCase(), display_name: body.display_name ?? '', disabled: false, bounce_count: 0 } + const address = body.address.toLowerCase() + const publisher = body.publisher ?? ms.find((x) => x.address === address)?.publisher ?? false + const mem: Member = { address, display_name: body.display_name ?? '', disabled: false, bounce_count: 0, publisher } state.members[id] = [...ms.filter((x) => x.address !== mem.address), mem] return json(201, mem) } } + if ((m = path.match(/^\/lists\/(\d+)\/members\/(.+)$/)) && method === 'PATCH') { + const mem = (state.members[Number(m[1])] ?? []).find((x) => x.address === decodeURIComponent(m![2]).toLowerCase()) + if (!mem) return err(404, 'not found') + if (typeof body.publisher !== 'boolean') return err(400, 'publisher (true or false) is required') + mem.publisher = body.publisher + return json(204) + } if ((m = path.match(/^\/lists\/(\d+)\/members\/(.+)$/)) && method === 'DELETE') { const id = Number(m[1]) const addr = decodeURIComponent(m[2]) @@ -125,6 +135,11 @@ export function installFakeApi(state: FakeState) { state.members[id] = before.filter((x) => x.address !== addr) return json(204) } + if ((m = path.match(/^\/lists\/(\d+)\/posts$/)) && method === 'GET') { + const id = Number(m[1]) + if (!state.lists.some((l) => l.id === id)) return err(404, 'not found') + return json(200, state.posts[id] ?? []) + } if ((m = path.match(/^\/lists\/(\d+)\/held$/)) && method === 'GET') return json(200, state.held[Number(m[1])] ?? []) if ((m = path.match(/^\/held\/(\d+)(\/approve)?$/))) { const id = Number(m[1]) diff --git a/docs/openapi.yaml b/docs/openapi.yaml index ad0b348..3347147 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -20,7 +20,7 @@ components: domain: {type: string} title: {type: string} address: {type: string} - post_policy: {type: string, enum: [members, moderated]} + post_policy: {type: string, enum: [members, moderated, publishers], description: "publishers: a newsletter, only members flagged publisher may post"} join_policy: {type: string, enum: [open, closed]} archive_policy: {type: string, enum: [public, private, off], description: "public: browsable on the web; private: API only (default); off: not stored"} Message: @@ -52,6 +52,7 @@ components: display_name: {type: string} disabled: {type: boolean, description: Disabled after 3 hard bounces within 30 days; re-adding the member re-enables it} bounce_count: {type: integer, description: Recent hard bounces (restarts at 1 when the last one is older than 30 days)} + publisher: {type: boolean, description: May post when the list's post_policy is publishers} Held: type: object properties: @@ -142,7 +143,7 @@ paths: type: object properties: title: {type: string, minLength: 1, maxLength: 200, description: A single line without control characters.} - post_policy: {type: string, enum: [members, moderated]} + post_policy: {type: string, enum: [members, moderated, publishers], description: "publishers: a newsletter, only members flagged publisher may post"} join_policy: {type: string, enum: [open, closed]} archive_policy: {type: string, enum: [public, private, off]} responses: @@ -182,6 +183,7 @@ paths: properties: address: {type: string} display_name: {type: string} + publisher: {type: boolean, description: "Absent: an existing member keeps its flag (a new one is not a publisher); true or false: set it"} responses: '201': description: Created @@ -199,6 +201,21 @@ paths: responses: '204': {description: Removed} '404': {$ref: '#/components/responses/Error'} + patch: + summary: Make a member a publisher or a plain subscriber + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [publisher] + properties: + publisher: {type: boolean} + responses: + '204': {description: Updated} + '400': {$ref: '#/components/responses/Error'} + '404': {$ref: '#/components/responses/Error'} /lists/{id}/held: parameters: - {name: id, in: path, required: true, schema: {type: integer}} @@ -287,6 +304,7 @@ paths: - {name: id, in: path, required: true, schema: {type: integer}} delete: summary: Discard a queued delivery + description: A waiting delivery of a post counts as dropped in the post's counters; a failed one stays counted as failed. responses: '204': {description: Discarded} '404': {$ref: '#/components/responses/Error'} @@ -319,3 +337,34 @@ paths: errors: {type: array, items: {type: string}, description: The first 20 failures} '400': {$ref: '#/components/responses/Error'} '404': {$ref: '#/components/responses/Error'} + /lists/{id}/posts: + parameters: + - {name: id, in: path, required: true, schema: {type: integer}} + get: + summary: Posts queued to the list with their delivery counters, newest first + description: > + One entry per post queued to the list (delivered or approved), kept for 400 days. Confirmation mail is not a post. + Bodies are never read. + parameters: + - {name: limit, in: query, schema: {type: integer, default: 50, maximum: 200}, description: 0 means the default; more than 200 gives 200} + responses: + '200': + description: OK + content: + application/json: + schema: + type: array + items: + type: object + properties: + id: {type: integer} + message_id: {type: string, description: The Message-ID header, at most 200 characters; empty when there was none} + subject: {type: string, description: Decoded, at most 200 characters, control characters removed} + created_at: {type: string, format: date-time} + recipients: {type: integer, description: Deliveries queued (members at the time of the post)} + sent: {type: integer, description: Accepted by the relay} + failed: {type: integer, description: Given up for good (refused by the relay, or still failing after 3 days)} + dropped: {type: integer, description: The mailbox does not exist, or the delivery was discarded} + pending: {type: integer, description: "Still in the queue: recipients - sent - failed - dropped"} + '400': {$ref: '#/components/responses/Error'} + '404': {$ref: '#/components/responses/Error'} diff --git a/internal/api/api.go b/internal/api/api.go index eb03fd7..ce8ed0b 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -68,6 +68,7 @@ func (s *server) routes() []route { {"GET /api/v1/lists/{id}/members", s.auth(s.listMembers)}, {"POST /api/v1/lists/{id}/members", s.auth(s.addMember)}, {"DELETE /api/v1/lists/{id}/members/{address}", s.auth(s.removeMember)}, + {"PATCH /api/v1/lists/{id}/members/{address}", s.auth(s.updateMember)}, {"GET /api/v1/lists/{id}/archive", s.auth(s.listArchive)}, {"POST /api/v1/lists/{id}/archive/import", s.auth(s.importArchive)}, {"GET /api/v1/archive/{id}", s.auth(s.getArchived)}, @@ -75,6 +76,7 @@ func (s *server) routes() []route { {"GET /api/v1/queue", s.auth(s.getQueue)}, {"DELETE /api/v1/queue/{id}", s.auth(s.discardQueued)}, {"GET /api/v1/lists/{id}/held", s.auth(s.listHeld)}, + {"GET /api/v1/lists/{id}/posts", s.auth(s.listPosts)}, {"POST /api/v1/held/{id}/approve", s.auth(s.approveHeld)}, {"DELETE /api/v1/held/{id}", s.auth(s.discardHeld)}, } @@ -124,6 +126,7 @@ type memberDTO struct { DisplayName string `json:"display_name"` Disabled bool `json:"disabled"` BounceCount int `json:"bounce_count"` + Publisher bool `json:"publisher"` } // Helpers. @@ -322,10 +325,10 @@ func (s *server) updateList(w http.ResponseWriter, r *http.Request) { l.ArchivePolicy = *in.ArchivePolicy } // Validate before writing anything. - if (l.PostPolicy != store.PostMembers && l.PostPolicy != store.PostModerated) || + if (l.PostPolicy != store.PostMembers && l.PostPolicy != store.PostModerated && l.PostPolicy != store.PostPublishers) || (l.JoinPolicy != store.JoinOpen && l.JoinPolicy != store.JoinClosed) || (l.ArchivePolicy != store.ArchivePublic && l.ArchivePolicy != store.ArchivePrivate && l.ArchivePolicy != store.ArchiveOff) { - writeError(w, http.StatusBadRequest, "post_policy must be members or moderated; join_policy open or closed; archive_policy public, private or off") + writeError(w, http.StatusBadRequest, "post_policy must be members, moderated or publishers; join_policy open or closed; archive_policy public, private or off") return } if err := s.store.UpdateList(id, l.Title); err != nil { @@ -369,7 +372,7 @@ func (s *server) listMembers(w http.ResponseWriter, r *http.Request) { } out := make([]memberDTO, 0, len(ms)) for _, m := range ms { - out = append(out, memberDTO{Address: m.Address, DisplayName: m.DisplayName, Disabled: m.Disabled, BounceCount: m.BounceCount}) + out = append(out, memberDTO{Address: m.Address, DisplayName: m.DisplayName, Disabled: m.Disabled, BounceCount: m.BounceCount, Publisher: m.Publisher}) } writeJSON(w, http.StatusOK, out) } @@ -379,7 +382,11 @@ func (s *server) addMember(w http.ResponseWriter, r *http.Request) { if !ok { return } - var in memberDTO + // publisher is optional: absent leaves an existing member's flag as it is. + var in struct { + memberDTO + Publisher *bool `json:"publisher"` + } if !decodeBody(w, r, &in) { return } @@ -400,7 +407,41 @@ func (s *server) addMember(w http.ResponseWriter, r *http.Request) { storeError(w, err) return } - writeJSON(w, http.StatusCreated, memberDTO{Address: strings.ToLower(a.Address), DisplayName: in.DisplayName}) + if in.Publisher != nil { + if err := s.store.SetPublisher(id, a.Address, *in.Publisher); err != nil { + storeError(w, err) + return + } + } + pub, err := s.store.IsPublisher(id, a.Address) + if err != nil { + storeError(w, err) + return + } + writeJSON(w, http.StatusCreated, memberDTO{Address: strings.ToLower(a.Address), DisplayName: in.DisplayName, Publisher: pub}) +} + +// updateMember changes the publisher flag of a member. +func (s *server) updateMember(w http.ResponseWriter, r *http.Request) { + id, ok := listID(w, r) + if !ok { + return + } + var in struct { + Publisher *bool `json:"publisher"` + } + if !decodeBody(w, r, &in) { + return + } + if in.Publisher == nil { + writeError(w, http.StatusBadRequest, "publisher (true or false) is required") + return + } + if err := s.store.SetPublisher(id, r.PathValue("address"), *in.Publisher); err != nil { + storeError(w, err) + return + } + w.WriteHeader(http.StatusNoContent) } func (s *server) removeMember(w http.ResponseWriter, r *http.Request) { @@ -627,7 +668,7 @@ func (s *server) discardQueued(w http.ResponseWriter, r *http.Request) { if !ok { return } - if err := s.store.CompleteOutbox(id); err != nil { + if err := s.store.DropOutbox(id); err != nil { storeError(w, err) return } @@ -661,3 +702,45 @@ func (s *server) importArchive(w http.ResponseWriter, r *http.Request) { } writeJSON(w, http.StatusOK, sum) } + +// Handlers: posts. + +type postDTO struct { + ID int64 `json:"id"` + MessageID string `json:"message_id"` + Subject string `json:"subject"` + CreatedAt time.Time `json:"created_at"` + Recipients int `json:"recipients"` + Sent int `json:"sent"` + Failed int `json:"failed"` + Dropped int `json:"dropped"` + Pending int `json:"pending"` +} + +// listPosts returns the posts queued to a list with their delivery counters, newest first. +func (s *server) listPosts(w http.ResponseWriter, r *http.Request) { + id, ok := listID(w, r) + if !ok { + return + } + limit, ok := intParam(r, "limit", 0) // 0: the store's default + if !ok { + writeError(w, http.StatusBadRequest, "limit must be a non-negative integer") + return + } + if _, err := s.store.GetList(id); err != nil { + storeError(w, err) + return + } + ps, err := s.store.ListPosts(id, limit) + if err != nil { + storeError(w, err) + return + } + out := make([]postDTO, 0, len(ps)) + for _, p := range ps { + out = append(out, postDTO{ID: p.ID, MessageID: p.MessageID, Subject: p.Subject, CreatedAt: p.CreatedAt, + Recipients: p.Recipients, Sent: p.Sent, Failed: p.Failed, Dropped: p.Dropped, Pending: p.Pending}) + } + writeJSON(w, http.StatusOK, out) +} diff --git a/internal/api/posts_test.go b/internal/api/posts_test.go new file mode 100644 index 0000000..1e1b9dc --- /dev/null +++ b/internal/api/posts_test.go @@ -0,0 +1,103 @@ +package api + +import ( + "fmt" + "strings" + "testing" + "time" + + "codeberg.org/squidd/parsimony/internal/store" +) + +type postJSON struct { + ID int64 `json:"id"` + MessageID string `json:"message_id"` + Subject string `json:"subject"` + CreatedAt time.Time `json:"created_at"` + Recipients int `json:"recipients"` + Sent int `json:"sent"` + Failed int `json:"failed"` + Dropped int `json:"dropped"` + Pending int `json:"pending"` +} + +func TestListPosts(t *testing.T) { + h, s := queueServer(t) + now := time.Unix(1_700_000_000, 0) + s.Now = func() time.Time { return now } + l, _ := s.CreateList("dev", "example.org", "Dev") + rcpts := []store.NewOutbound{{From: "b@x.org", Rcpt: "a@x.com"}, {From: "b@x.org", Rcpt: "b@x.com"}, {From: "b@x.org", Rcpt: "c@x.com"}} + s.EnqueuePost(l.ID, "mid:<1@x>", []byte("Message-ID: <1@x>\r\nSubject: first\r\n\r\nsecret body"), rcpts) + now = now.Add(time.Minute) + s.EnqueuePost(l.ID, "mid:<2@x>", []byte("Message-ID: <2@x>\r\nSubject: second\r\n\r\nsecret body"), rcpts[:1]) + due, _ := s.DueOutbox(10) + s.CompleteOutbox(due[0].ID) + s.FailOutbox(due[1].ID, "no") + + rec := do(t, h, "GET", fmt.Sprintf("/api/v1/lists/%d/posts", l.ID), "", true) + expect(t, rec, 200) + var ps []postJSON + decode(t, rec, &ps) + if len(ps) != 2 || ps[0].Subject != "second" || ps[1].Subject != "first" { + t.Fatalf("newest first: %+v", ps) + } + p := ps[1] + if p.ID == 0 || p.MessageID != "<1@x>" || !p.CreatedAt.Equal(now.Add(-time.Minute)) || p.Recipients != 3 || + p.Sent != 1 || p.Failed != 1 || p.Dropped != 0 || p.Pending != 1 { + t.Fatalf("post: %+v", p) + } + if strings.Contains(rec.Body.String(), "secret body") { + t.Fatal("bodies are never listed") + } + + decode(t, do(t, h, "GET", fmt.Sprintf("/api/v1/lists/%d/posts?limit=1", l.ID), "", true), &ps) + if len(ps) != 1 || ps[0].Subject != "second" { + t.Fatalf("limit=1: %+v", ps) + } +} + +func TestListPostsEmptyIsAnArray(t *testing.T) { + h, s := queueServer(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + rec := do(t, h, "GET", fmt.Sprintf("/api/v1/lists/%d/posts", l.ID), "", true) + expect(t, rec, 200) + if strings.TrimSpace(rec.Body.String()) != "[]" { + t.Fatalf("body %q", rec.Body.String()) + } +} + +func TestListPostsLimit(t *testing.T) { + h, s := queueServer(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + for i := 0; i < 60; i++ { + s.EnqueuePost(l.ID, fmt.Sprintf("mid:%d", i), []byte("Subject: x\r\n\r\n"), nil) + } + path := fmt.Sprintf("/api/v1/lists/%d/posts", l.ID) + var ps []postJSON + decode(t, do(t, h, "GET", path, "", true), &ps) + if len(ps) != 50 { + t.Fatalf("default 50: %d", len(ps)) + } + for _, bad := range []string{"-1", "x", "1.5"} { + expect(t, do(t, h, "GET", path+"?limit="+bad, "", true), 400) + } +} + +func TestListPostsUnknownListAndAuth(t *testing.T) { + h, _ := queueServer(t) + expect(t, do(t, h, "GET", "/api/v1/lists/99/posts", "", true), 404) + expect(t, do(t, h, "GET", "/api/v1/lists/abc/posts", "", true), 404) + expect(t, do(t, h, "GET", "/api/v1/lists/99/posts", "", false), 401) +} + +func TestDiscardingADeliveryOfAPostCountsAsDropped(t *testing.T) { + h, s := queueServer(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.EnqueuePost(l.ID, "mid:<1@x>", []byte("Subject: x\r\n\r\n"), []store.NewOutbound{{From: "b@x.org", Rcpt: "a@x.com"}}) + due, _ := s.DueOutbox(10) + expect(t, do(t, h, "DELETE", fmt.Sprintf("/api/v1/queue/%d", due[0].ID), "", true), 204) + ps, _ := s.ListPosts(l.ID, 0) + if len(ps) != 1 || ps[0].Dropped != 1 || ps[0].Sent != 0 || ps[0].Pending != 0 { + t.Fatalf("posts: %+v", ps) + } +} diff --git a/internal/api/publisher_test.go b/internal/api/publisher_test.go new file mode 100644 index 0000000..49cfe03 --- /dev/null +++ b/internal/api/publisher_test.go @@ -0,0 +1,112 @@ +package api + +import ( + "net/http" + "strings" + "testing" +) + +type publisherJSON struct { + Address string `json:"address"` + Publisher bool `json:"publisher"` +} + +// publishers returns the publisher flag of every member of list 1, by address. +func publishers(t *testing.T, h http.Handler) map[string]bool { + t.Helper() + rec := do(t, h, "GET", "/api/v1/lists/1/members", "", true) + expect(t, rec, 200) + var ms []publisherJSON + decode(t, rec, &ms) + out := map[string]bool{} + for _, m := range ms { + out[m.Address] = m.Publisher + } + return out +} + +func TestAddMemberWithThePublisherFlag(t *testing.T) { + h := newServer(t) + createList(t, h) + rec := do(t, h, "POST", "/api/v1/lists/1/members", `{"address":"a@x.com","publisher":true}`, true) + expect(t, rec, 201) + var m publisherJSON + decode(t, rec, &m) + if !m.Publisher { + t.Fatalf("created: %+v", m) + } + expect(t, do(t, h, "POST", "/api/v1/lists/1/members", `{"address":"b@x.com"}`, true), 201) + if got := publishers(t, h); !got["a@x.com"] || got["b@x.com"] { + t.Fatalf("publishers: %v", got) + } +} + +func TestAddMemberLeavesThePublisherFlagAloneWhenAbsent(t *testing.T) { + h := newServer(t) + createList(t, h) + expect(t, do(t, h, "POST", "/api/v1/lists/1/members", `{"address":"a@x.com","publisher":true}`, true), 201) + rec := do(t, h, "POST", "/api/v1/lists/1/members", `{"address":"a@x.com","display_name":"A"}`, true) + expect(t, rec, 201) + var m publisherJSON + decode(t, rec, &m) + if !m.Publisher || !publishers(t, h)["a@x.com"] { + t.Fatalf("re-adding keeps the flag: %+v %v", m, publishers(t, h)) + } + // An explicit false demotes an existing member. + rec = do(t, h, "POST", "/api/v1/lists/1/members", `{"address":"a@x.com","publisher":false}`, true) + expect(t, rec, 201) + decode(t, rec, &m) + if m.Publisher || publishers(t, h)["a@x.com"] { + t.Fatalf("demoted: %+v %v", m, publishers(t, h)) + } +} + +func TestPatchMemberSetsThePublisherFlag(t *testing.T) { + h := newServer(t) + createList(t, h) + expect(t, do(t, h, "POST", "/api/v1/lists/1/members", `{"address":"a@x.com"}`, true), 201) + expect(t, do(t, h, "PATCH", "/api/v1/lists/1/members/A%40X.com", `{"publisher":true}`, true), 204) + if !publishers(t, h)["a@x.com"] { + t.Fatal("promoted") + } + expect(t, do(t, h, "PATCH", "/api/v1/lists/1/members/a%40x.com", `{"publisher":false}`, true), 204) + if publishers(t, h)["a@x.com"] { + t.Fatal("demoted") + } +} + +func TestPatchMemberErrors(t *testing.T) { + h := newServer(t) + createList(t, h) + expect(t, do(t, h, "POST", "/api/v1/lists/1/members", `{"address":"a@x.com"}`, true), 201) + expect(t, do(t, h, "PATCH", "/api/v1/lists/1/members/b%40x.com", `{"publisher":true}`, true), 404) + expect(t, do(t, h, "PATCH", "/api/v1/lists/99/members/a%40x.com", `{"publisher":true}`, true), 404) + rec := do(t, h, "PATCH", "/api/v1/lists/1/members/a%40x.com", `{}`, true) + expect(t, rec, 400) + if !strings.Contains(rec.Body.String(), "publisher") { + t.Fatalf("the error names the field: %s", rec.Body.String()) + } + expect(t, do(t, h, "PATCH", "/api/v1/lists/1/members/a%40x.com", `{"publisher":"yes"}`, true), 400) + expect(t, do(t, h, "PATCH", "/api/v1/lists/1/members/a%40x.com", `{"publisher":true,"x":1}`, true), 400) + expect(t, do(t, h, "PATCH", "/api/v1/lists/1/members/a%40x.com", `{"publisher":true}`, false), 401) + if publishers(t, h)["a@x.com"] { + t.Fatal("nothing changed") + } +} + +func TestThePublishersPostPolicyViaAPI(t *testing.T) { + h := newServer(t) + createList(t, h) + rec := do(t, h, "PATCH", "/api/v1/lists/1", `{"post_policy":"publishers"}`, true) + expect(t, rec, 200) + var l listJSON + decode(t, rec, &l) + if l.PostPolicy != "publishers" { + t.Fatalf("patched: %+v", l) + } + rec = do(t, h, "PATCH", "/api/v1/lists/1", `{"post_policy":"bogus"}`, true) + expect(t, rec, 400) + if !strings.Contains(rec.Body.String(), "members, moderated or publishers") { + t.Fatalf("the message lists the policies: %s", rec.Body.String()) + } +} diff --git a/internal/app/app.go b/internal/app/app.go index 5115942..3c8e2ca 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -229,6 +229,7 @@ func Start(cfg config.Config, apiToken string) (*App, error) { Username: cfg.Relay.Username, Password: relayPassword, Helo: heloName(cfg.Relay.Helo), }, Concurrency: cfg.Relay.Concurrency, + Rate: float64(cfg.Relay.RatePerSecond), Sign: sign, } rt.OnEnqueue = worker.Wake diff --git a/internal/app/app_test.go b/internal/app/app_test.go index e96ea79..92f4410 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -1310,3 +1310,52 @@ func TestHTTPServersCarryTheTimeouts(t *testing.T) { } } } + +func TestTheRelayRateReachesTheWorker(t *testing.T) { + relayAddr, relay := startSink(t) + cfg := testConfig(t, relayAddr) + cfg.Relay.RatePerSecond = 20 // 50 ms between two sends + a, err := Start(cfg, "tok") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { a.Close() }) + call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org"}`, "tok") + const members = 8 + for i := 0; i < members; i++ { + call(t, a, "POST", "/api/v1/lists/1/members", fmt.Sprintf(`{"address":"m%d@example.com"}`, i), "tok") + } + start := time.Now() + msg := "From: M0 \r\nSubject: paced\r\nMessage-ID: \r\n\r\nbody\r\n" + if err := lmtpSend(t, a.LMTPAddr, "m0@example.com", "dev@example.org", msg); err != nil { + t.Fatal(err) + } + relay.waitCount(t, members) + // 8 sends started 50 ms apart: the last one cannot start before 350 ms. + if took := time.Since(start); took < 330*time.Millisecond { + t.Fatalf("%d deliveries took %v: the rate did not reach the worker", members, took) + } +} + +func TestWithoutARateTheRelayIsNotPaced(t *testing.T) { + relayAddr, relay := startSink(t) + a, err := Start(testConfig(t, relayAddr), "tok") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { a.Close() }) + call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org"}`, "tok") + const members = 8 + for i := 0; i < members; i++ { + call(t, a, "POST", "/api/v1/lists/1/members", fmt.Sprintf(`{"address":"m%d@example.com"}`, i), "tok") + } + start := time.Now() + msg := "From: M0 \r\nSubject: free\r\nMessage-ID: \r\n\r\nbody\r\n" + if err := lmtpSend(t, a.LMTPAddr, "m0@example.com", "dev@example.org", msg); err != nil { + t.Fatal(err) + } + relay.waitCount(t, members) + if took := time.Since(start); took > 3*time.Second { + t.Fatalf("%d deliveries took %v without a rate", members, took) + } +} diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 2d50292..9118c33 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -29,15 +29,20 @@ Administration (talks to the REST API): list ls show all lists list add [--title T] create a list list show show one list - list set [--title T] [--post members|moderated] + list set [--title T] [--post members|moderated|publishers] [--join open|closed] [--archive public|private|off] list rm --yes delete a list with its members and archive - member ls [--emails] show subscribers - member add [--name N] subscribe (or re-enable) an address + member ls [--emails] show subscribers and publishers + member add [--name N] [--publisher] + subscribe (or re-enable) an address; + --publisher lets it post to a publishers list, + --publisher=false takes that away + member set --publisher=true|false + make a member a publisher or a plain subscriber member rm unsubscribe an address member import subscribe addresses from a file or stdin, - one "email[,name]" per line + one "email[,name[,publisher]]" per line held ls messages waiting for moderation held approve deliver a held message @@ -49,6 +54,8 @@ Administration (talks to the REST API): archive import add the messages of an mbox file (Mailman export); safe to repeat, threads are rebuilt + posts ls [--limit N] posts sent to a list with their delivery counts + queue ls outbound queue queue rm discard one queued delivery @@ -101,10 +108,11 @@ func Run(args []string, io IO, getenv func(string) string) int { commands := map[string]map[string]func([]string) int{ "list": {"ls": c.listLs, "add": c.listAdd, "show": c.listShow, "set": c.listSet, "rm": c.listRm}, - "member": {"ls": c.memberLs, "add": c.memberAdd, "rm": c.memberRm, "import": c.memberImport}, + "member": {"ls": c.memberLs, "add": c.memberAdd, "set": c.memberSet, "rm": c.memberRm, "import": c.memberImport}, "held": {"ls": c.heldLs, "approve": c.heldApprove, "discard": c.heldDiscard}, "archive": {"ls": c.archiveLs, "show": c.archiveShow, "rm": c.archiveRm, "import": c.archiveImport}, "queue": {"ls": c.queueLs, "rm": c.queueRm}, + "posts": {"ls": c.postsLs}, "dkim": {"keygen": c.dkimKeygen, "record": c.dkimRecord, "check": c.dkimCheck}, } group, ok := commands[rest[0]] diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index b834f6b..3754b25 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -274,7 +274,7 @@ func TestMembers(t *testing.T) { func TestMemberImport(t *testing.T) { e := newEnv(t) e.ok(t, "list", "add", "dev@example.org") - input := "# subscribers exported from Mailman\n\nalice@example.com, Alice Smith\nbob@example.com\n carol@example.com ,Carol \nnot-an-address\ndave@example.com,Dave,extra\n" + input := "# subscribers exported from Mailman\n\nalice@example.com, Alice Smith\nbob@example.com\n carol@example.com ,Carol \nnot-an-address\ndave@example.com,Dave,Publisher\n" code, out, errs := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}, input, "member", "import", "dev@example.org", "-") if code != 1 { t.Fatalf("a failed line must make the exit code non-zero, got %d", code) diff --git a/internal/cli/commands.go b/internal/cli/commands.go index 1232346..bf663f3 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -179,6 +179,14 @@ type member struct { DisplayName string `json:"display_name"` Disabled bool `json:"disabled"` BounceCount int `json:"bounce_count"` + Publisher bool `json:"publisher"` +} + +func (m member) role() string { + if m.Publisher { + return "publisher" + } + return "subscriber" } func (m member) status() string { @@ -228,9 +236,9 @@ func (c *cmd) memberLs(args []string) int { fmt.Fprintln(c.io.Out, "No members yet.") return 0 } - rows := [][]string{{"ADDRESS", "NAME", "STATUS"}} + rows := [][]string{{"ADDRESS", "NAME", "ROLE", "STATUS"}} for _, m := range ms { - rows = append(rows, []string{m.Address, m.DisplayName, m.status()}) + rows = append(rows, []string{m.Address, m.DisplayName, m.role(), m.status()}) } c.table(rows) return 0 @@ -239,18 +247,19 @@ func (c *cmd) memberLs(args []string) int { func (c *cmd) memberAdd(args []string) int { fs := newFlagSet("member add") name := fs.String("name", "", "") + pub := fs.Bool("publisher", false, "") pos, err := parseArgs(fs, args) if err != nil { return c.usageError("%v", err) } if len(pos) != 2 { - return c.usageError("usage: parsimony member add [--name NAME]") + return c.usageError("usage: parsimony member add [--name NAME] [--publisher]") } l, err := c.resolveList(pos[0]) if err != nil { return c.fail(err) } - raw, err := c.addMember(l.ID, pos[1], *name) + raw, err := c.addMember(l.ID, pos[1], *name, flagBool(fs, "publisher", *pub)) if err != nil { return c.fail(err) } @@ -259,16 +268,65 @@ func (c *cmd) memberAdd(args []string) int { } var m member json.Unmarshal(raw, &m) - fmt.Fprintf(c.io.Out, "Added %s to %s\n", printable(m.Address), printable(l.Address)) + as := "" + if m.Publisher { + as = " as a publisher" + } + fmt.Fprintf(c.io.Out, "Added %s to %s%s\n", printable(m.Address), printable(l.Address), as) return 0 } -func (c *cmd) addMember(listID int64, address, name string) (json.RawMessage, error) { +// flagBool returns &v when the flag was given on the command line, nil otherwise. +func flagBool(fs *flag.FlagSet, name string, v bool) *bool { + set := false + fs.Visit(func(f *flag.Flag) { set = set || f.Name == name }) + if !set { + return nil + } + return &v +} + +// addMember subscribes an address; a nil publisher leaves the flag of an existing member alone. +func (c *cmd) addMember(listID int64, address, name string, publisher *bool) (json.RawMessage, error) { cl, err := c.client() if err != nil { return nil, err } - return cl.do("POST", fmt.Sprintf("/lists/%d/members", listID), map[string]string{"address": address, "display_name": name}) + body := map[string]any{"address": address, "display_name": name} + if publisher != nil { + body["publisher"] = *publisher + } + return cl.do("POST", fmt.Sprintf("/lists/%d/members", listID), body) +} + +func (c *cmd) memberSet(args []string) int { + fs := newFlagSet("member set") + pub := fs.Bool("publisher", false, "") + pos, err := parseArgs(fs, args) + if err != nil { + return c.usageError("%v", err) + } + if len(pos) != 2 { + return c.usageError("usage: parsimony member set --publisher=true|false") + } + publisher := flagBool(fs, "publisher", *pub) + if publisher == nil { + return c.usageError("nothing to change: give --publisher=true or --publisher=false") + } + l, err := c.resolveList(pos[0]) + if err != nil { + return c.fail(err) + } + cl, _ := c.client() + path := fmt.Sprintf("/lists/%d/members/%s", l.ID, url.PathEscape(pos[1])) + if _, err := cl.do("PATCH", path, map[string]bool{"publisher": *publisher}); err != nil { + return c.fail(err) + } + role := "a subscriber" + if *publisher { + role = "a publisher" + } + return c.done(fmt.Sprintf("%s is now %s of %s", strings.ToLower(pos[1]), role, l.Address)) } func (c *cmd) memberRm(args []string) int { @@ -325,7 +383,20 @@ func (c *cmd) memberImport(args []string) int { if len(parts) > 1 { name = strings.TrimSpace(parts[1]) } - if _, err := c.addMember(l.ID, address, name); err != nil { + // The optional third column is the keyword "publisher"; an empty one is no column. + var publisher *bool + if len(parts) > 2 { + switch kw := strings.TrimSpace(parts[2]); { + case strings.EqualFold(kw, "publisher"): + yes := true + publisher = &yes + case kw != "": + failed++ + fmt.Fprintf(c.io.Err, "line %d: %s\n", n, printable(fmt.Sprintf("%s: the third column may only be \"publisher\"", line))) + continue + } + } + if _, err := c.addMember(l.ID, address, name, publisher); err != nil { failed++ fmt.Fprintf(c.io.Err, "line %d: %s\n", n, printable(fmt.Sprintf("%s: %v", line, err))) continue @@ -683,3 +754,56 @@ func (c *cmd) queueRm(args []string) int { return err }, "Discarded queued delivery %d") } + +// postsLs shows the posts queued to a list, newest first, with how far their delivery got. +func (c *cmd) postsLs(args []string) int { + fs := newFlagSet("posts ls") + limit := fs.Int("limit", 50, "") + pos, err := parseArgs(fs, args) + if err != nil { + return c.usageError("%v", err) + } + if len(pos) != 1 { + return c.usageError("usage: parsimony posts ls [--limit N]") + } + l, err := c.resolveList(pos[0]) + if err != nil { + return c.fail(err) + } + cl, _ := c.client() + raw, err := cl.do("GET", fmt.Sprintf("/lists/%d/posts?limit=%d", l.ID, *limit), nil) + if err != nil { + return c.fail(err) + } + if c.json { + return c.printJSON(raw) + } + var ps []struct { + ID int64 `json:"id"` + Subject string `json:"subject"` + CreatedAt time.Time `json:"created_at"` + Recipients int `json:"recipients"` + Sent int `json:"sent"` + Failed int `json:"failed"` + Dropped int `json:"dropped"` + Pending int `json:"pending"` + } + if err := json.Unmarshal(raw, &ps); err != nil { + return c.fail(err) + } + if len(ps) == 0 { + fmt.Fprintln(c.io.Out, "No posts yet.") + return 0 + } + rows := [][]string{{"ID", "DATE", "SUBJECT", "RECIPIENTS", "SENT", "PENDING", "FAILED", "DROPPED"}} + for _, p := range ps { + subject := p.Subject + if subject == "" { + subject = "(no subject)" + } + rows = append(rows, []string{strconv.FormatInt(p.ID, 10), formatDate(p.CreatedAt), subject, strconv.Itoa(p.Recipients), + strconv.Itoa(p.Sent), strconv.Itoa(p.Pending), strconv.Itoa(p.Failed), strconv.Itoa(p.Dropped)}) + } + c.table(rows) + return 0 +} diff --git a/internal/cli/posts_test.go b/internal/cli/posts_test.go new file mode 100644 index 0000000..ed8b597 --- /dev/null +++ b/internal/cli/posts_test.go @@ -0,0 +1,95 @@ +package cli + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "codeberg.org/squidd/parsimony/internal/store" +) + +func TestPostsLs(t *testing.T) { + e := newEnv(t) + e.ok(t, "list", "add", "dev@example.org") + l, _ := e.s.FindList("dev@example.org") + if out := e.ok(t, "posts", "ls", "dev@example.org"); !strings.Contains(out, "No posts yet.") { + t.Fatalf("empty: %q", out) + } + now := time.Date(2026, 3, 1, 9, 30, 0, 0, time.UTC) + e.s.Now = func() time.Time { return now } + var rcpts []store.NewOutbound + for i := 0; i < 7; i++ { + rcpts = append(rcpts, store.NewOutbound{From: "b@x.org", Rcpt: fmt.Sprintf("u%d@x.com", i)}) + } + e.s.EnqueuePost(l.ID, "mid:<1@x>", []byte("Message-ID: <1@x>\r\nSubject: March issue\r\n\r\nb"), rcpts) + due, _ := e.s.DueOutbox(10) + for _, d := range due[:4] { + e.s.CompleteOutbox(d.ID) + } + e.s.FailOutbox(due[4].ID, "no") + now = now.Add(time.Hour) + e.s.EnqueuePost(l.ID, "mid:<2@x>", []byte("Message-ID: <2@x>\r\n\r\nb"), nil) + + out := e.ok(t, "posts", "ls", fmt.Sprint(l.ID)) + lines := strings.Split(strings.TrimSpace(out), "\n") + if len(lines) != 3 { + t.Fatalf("posts ls:\n%s", out) + } + if f := strings.Fields(lines[0]); strings.Join(f, " ") != "ID DATE SUBJECT RECIPIENTS SENT PENDING FAILED DROPPED" { + t.Fatalf("header: %q", lines[0]) + } + if f := strings.Fields(lines[1]); strings.Join(f, " ") != "2 2026-03-01 10:30 (no subject) 0 0 0 0 0" { + t.Fatalf("newest first: %q", lines[1]) + } + if f := strings.Fields(lines[2]); strings.Join(f, " ") != "1 2026-03-01 09:30 March issue 7 4 2 1 0" { + t.Fatalf("row: %q", lines[2]) + } + + if out := e.ok(t, "posts", "ls", "dev@example.org", "--limit", "1"); strings.Contains(out, "March issue") || !strings.Contains(out, "(no subject)") { + t.Fatalf("--limit 1:\n%s", out) + } + var js []map[string]any + if err := json.Unmarshal([]byte(e.ok(t, "--json", "posts", "ls", "dev@example.org")), &js); err != nil || len(js) != 2 || + js[1]["subject"] != "March issue" || js[1]["pending"] != float64(2) || js[1]["message_id"] != "<1@x>" { + t.Fatalf("json: %v %v", js, err) + } +} + +func TestPostsLsErrors(t *testing.T) { + e := newEnv(t) + e.fails(t, 2, "usage: parsimony posts ls", "posts", "ls") + e.fails(t, 2, "usage: parsimony posts ls", "posts", "ls", "a@x", "b@x") + e.fails(t, 2, "flag provided but not defined", "posts", "ls", "a@x", "--nope") + e.fails(t, 1, "no such list", "posts", "ls", "nope@example.org") + e.ok(t, "list", "add", "dev@example.org") + e.fails(t, 1, "limit must be", "posts", "ls", "dev@example.org", "--limit", "-1") + if out := e.ok(t, "help"); !strings.Contains(out, "posts ls [--limit N]") { + t.Fatalf("help lacks posts ls:\n%s", out) + } +} + +func TestPostsLsEscapesControlCharacters(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/api/v1/lists/1": + fmt.Fprint(w, `{"id":1,"address":"dev@example.org"}`) + case "/api/v1/lists/1/posts": + b, _ := json.Marshal([]map[string]any{{"id": 1, "subject": "s " + esc, "created_at": time.Unix(0, 0), "recipients": 1}}) + w.Write(b) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + e := &env{srv: srv} + out := e.ok(t, "posts", "ls", "1") + noControl(t, "posts ls", out) + if !strings.Contains(out, `\x1b[31mRED`) { + t.Fatalf("the escape sequence should stay visible as text:\n%s", out) + } +} diff --git a/internal/cli/publisher_test.go b/internal/cli/publisher_test.go new file mode 100644 index 0000000..2cbacc8 --- /dev/null +++ b/internal/cli/publisher_test.go @@ -0,0 +1,144 @@ +package cli + +import ( + "strings" + "testing" +) + +// isPublisher asks the store whether addr is a publisher of dev@example.org. +func (e *env) isPublisher(t *testing.T, addr string) bool { + t.Helper() + l, err := e.s.FindList("dev@example.org") + if err != nil { + t.Fatal(err) + } + ok, err := e.s.IsPublisher(l.ID, addr) + if err != nil { + t.Fatal(err) + } + return ok +} + +func TestMemberAddWithThePublisherFlag(t *testing.T) { + e := newEnv(t) + e.ok(t, "list", "add", "dev@example.org") + out := e.ok(t, "member", "add", "dev@example.org", "alice@example.com", "--publisher") + if !strings.Contains(out, "Added alice@example.com") || !strings.Contains(out, "publisher") { + t.Fatalf("add: %q", out) + } + if !e.isPublisher(t, "alice@example.com") { + t.Fatal("--publisher makes a publisher") + } + // Without the flag an existing member keeps its role. + if out := e.ok(t, "member", "add", "dev@example.org", "alice@example.com", "--name", "Alice"); !strings.Contains(out, "publisher") { + t.Fatalf("re-add: %q", out) + } + if !e.isPublisher(t, "alice@example.com") { + t.Fatal("re-adding without the flag keeps it") + } + if out := e.ok(t, "member", "add", "dev@example.org", "alice@example.com", "--publisher=false"); strings.Contains(out, "publisher") { + t.Fatalf("demote: %q", out) + } + if e.isPublisher(t, "alice@example.com") { + t.Fatal("--publisher=false demotes") + } + e.ok(t, "member", "add", "dev@example.org", "bob@example.com") + if e.isPublisher(t, "bob@example.com") { + t.Fatal("a plain add is a subscriber") + } +} + +func TestMemberLsShowsTheRole(t *testing.T) { + e := newEnv(t) + e.ok(t, "list", "add", "dev@example.org") + e.ok(t, "member", "add", "dev@example.org", "alice@example.com", "--publisher") + e.ok(t, "member", "add", "dev@example.org", "bob@example.com") + out := e.ok(t, "member", "ls", "dev@example.org") + lines := strings.Split(strings.TrimSpace(out), "\n") + if len(lines) != 3 || !strings.Contains(lines[0], "ROLE") { + t.Fatalf("ls:\n%s", out) + } + if !strings.Contains(lines[1], "alice@example.com") || !strings.Contains(lines[1], "publisher") { + t.Fatalf("alice:\n%s", out) + } + if !strings.Contains(lines[2], "bob@example.com") || !strings.Contains(lines[2], "subscriber") { + t.Fatalf("bob:\n%s", out) + } + if out := e.ok(t, "--json", "member", "ls", "dev@example.org"); !strings.Contains(out, `"publisher": true`) { + t.Fatalf("json: %s", out) + } +} + +func TestMemberSetChangesTheRole(t *testing.T) { + e := newEnv(t) + e.ok(t, "list", "add", "dev@example.org") + e.ok(t, "member", "add", "dev@example.org", "alice@example.com") + out := e.ok(t, "member", "set", "dev@example.org", "Alice@Example.com", "--publisher=true") + if !strings.Contains(out, "alice@example.com") || !strings.Contains(out, "publisher") { + t.Fatalf("set: %q", out) + } + if !e.isPublisher(t, "alice@example.com") { + t.Fatal("promoted") + } + out = e.ok(t, "member", "set", "dev@example.org", "alice@example.com", "--publisher=false") + if !strings.Contains(out, "subscriber") { + t.Fatalf("set false: %q", out) + } + if e.isPublisher(t, "alice@example.com") { + t.Fatal("demoted") + } + e.ok(t, "member", "set", "dev@example.org", "alice@example.com", "--publisher") + if !e.isPublisher(t, "alice@example.com") { + t.Fatal("a bare --publisher means true") + } + e.fails(t, 2, "--publisher", "member", "set", "dev@example.org", "alice@example.com") + e.fails(t, 2, "usage: parsimony member set", "member", "set", "dev@example.org", "--publisher") + e.fails(t, 2, "invalid boolean", "member", "set", "dev@example.org", "alice@example.com", "--publisher=maybe") + e.fails(t, 1, "not found", "member", "set", "dev@example.org", "bob@example.com", "--publisher") + e.fails(t, 1, "no such list", "member", "set", "ghost@example.org", "alice@example.com", "--publisher") +} + +func TestMemberImportReadsThePublisherColumn(t *testing.T) { + e := newEnv(t) + e.ok(t, "list", "add", "dev@example.org") + e.ok(t, "member", "add", "dev@example.org", "erin@example.com", "--publisher") + input := "alice@example.com,Alice,publisher\nbob@example.com,, PUBLISHER \ncarol@example.com,Carol,admin\ndave@example.com,Dave,\nerin@example.com,Erin\nfrank@example.com,Frank,publisher,x\n" + code, out, errs := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}, input, "member", "import", "dev@example.org", "-") + if code != 1 || !strings.Contains(out, "Imported 4") || !strings.Contains(out, "2 failed") { + t.Fatalf("%d %q %q", code, out, errs) + } + if !strings.Contains(errs, "line 3") || !strings.Contains(errs, "admin") || !strings.Contains(errs, "line 6") { + t.Fatalf("the bad third columns are reported: %q", errs) + } + for addr, want := range map[string]bool{"alice@example.com": true, "bob@example.com": true, "dave@example.com": false, "erin@example.com": true} { + if got := e.isPublisher(t, addr); got != want { + t.Errorf("%s publisher = %v, want %v", addr, got, want) + } + } + l, _ := e.s.FindList("dev@example.org") + for _, addr := range []string{"carol@example.com", "frank@example.com"} { + if ok, _ := e.s.IsMember(l.ID, addr); ok { + t.Errorf("%s: a line with a bad third column imports nothing", addr) + } + } +} + +func TestListSetPostPublishers(t *testing.T) { + e := newEnv(t) + e.ok(t, "list", "add", "dev@example.org") + e.ok(t, "list", "set", "dev@example.org", "--post", "publishers") + if out := e.ok(t, "list", "ls"); !strings.Contains(out, "publishers") { + t.Fatalf("ls: %s", out) + } + e.fails(t, 1, "members, moderated or publishers", "list", "set", "dev@example.org", "--post", "bogus") +} + +func TestHelpMentionsPublishers(t *testing.T) { + e := newEnv(t) + out := e.ok(t, "help") + for _, want := range []string{"--post members|moderated|publishers", "member add [--name N] [--publisher]", "member set --publisher=true|false", "publisher"} { + if !strings.Contains(out, want) { + t.Errorf("help lacks %q", want) + } + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 569048d..7f8f8e8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -6,6 +6,7 @@ import ( "crypto/x509" "errors" "fmt" + "math" "net" "net/url" "os" @@ -37,6 +38,9 @@ type Config struct { Relay struct { Addr string `yaml:"addr"` // host:port of the outgoing SMTP relay Concurrency int `yaml:"concurrency"` // simultaneous deliveries + // RatePerSecond is the most sends started per second, for relays that throttle (Amazon + // SES). Zero, the default, is unlimited. + RatePerSecond Rate `yaml:"rate_per_second"` // TLS is "none" (default: a local Postfix), "starttls" (required, never silently skipped) // or "tls" (encrypted from the first byte, port 465). TLS string `yaml:"tls"` @@ -50,6 +54,20 @@ type Config struct { } `yaml:"relay"` } +// Rate is a number of sends per second. It exists so that a value that is not a number is reported +// with the name of the key, as the other checks are. +type Rate float64 + +// UnmarshalYAML reads a YAML number. +func (r *Rate) UnmarshalYAML(n *yaml.Node) error { + var f float64 + if err := n.Decode(&f); err != nil { + return fmt.Errorf("line %d: relay.rate_per_second must be a number of sends per second, not %q", n.Line, n.Value) + } + *r = Rate(f) + return nil +} + // DKIMKey is the signing key of one domain. type DKIMKey struct { Domain string `yaml:"domain"` @@ -138,6 +156,9 @@ func (c Config) validateRelay() error { default: return fmt.Errorf("relay.tls must be none, starttls or tls, not %q", r.TLS) } + if math.IsNaN(float64(r.RatePerSecond)) || math.IsInf(float64(r.RatePerSecond), 0) || r.RatePerSecond < 0 { + return fmt.Errorf("relay.rate_per_second must be a number of sends per second, 0 or more (0 is unlimited), not %v", r.RatePerSecond) + } if (r.Username == "") != (r.PasswordFile == "") { return errors.New("relay.username and relay.password_file go together") } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 9a27087..5ff9076 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -170,6 +170,28 @@ func TestLoadRejectsBadRelaySettings(t *testing.T) { } } +func TestRelayRatePerSecond(t *testing.T) { + c, err := Load(write(t, "{}\n")) + if err != nil || float64(c.Relay.RatePerSecond) != 0 { + t.Fatalf("default is unlimited: %v %v", c.Relay.RatePerSecond, err) + } + for y, want := range map[string]float64{"14": 14, "0.5": 0.5, "0": 0, "0.0": 0, "1e2": 100} { + c, err := Load(write(t, "relay:\n rate_per_second: "+y+"\n")) + if err != nil || float64(c.Relay.RatePerSecond) != want { + t.Errorf("rate_per_second %s: got %v, %v; want %v", y, c.Relay.RatePerSecond, err, want) + } + } +} + +func TestLoadRejectsABadRelayRate(t *testing.T) { + for _, v := range []string{"-1", "-0.5", ".nan", ".NaN", ".inf", "-.inf", `"nan"`, `"fast"`, "fast", "nan", "[]", "{}"} { + _, err := Load(write(t, "relay:\n rate_per_second: "+v+"\n")) + if err == nil || !strings.Contains(err.Error(), "relay.rate_per_second") { + t.Errorf("rate_per_second %s: want an error naming the key, got %v", v, err) + } + } +} + func TestCredentialsToALoopbackRelayNeedNoTLS(t *testing.T) { for _, addr := range []string{"127.0.0.1:587", "localhost:587", "[::1]:587"} { if _, err := Load(write(t, "relay:\n addr: \""+addr+"\"\n username: u\n password_file: /p\n")); err != nil { diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 42a0e20..953300f 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -96,6 +96,8 @@ func (s *session) status(rcpt string, err error) error { return &smtp.SMTPError{Code: 552, EnhancedCode: smtp.EnhancedCode{5, 3, 4}, Message: "message too large for this moderated list"} case errors.Is(err, router.ErrNotMember): return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "only list members may post"} + case errors.Is(err, router.ErrNotPublisher): + return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "only the publishers of this list may post"} case errors.Is(err, router.ErrBadToken): return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "invalid or expired token"} case errors.Is(err, router.ErrJoinClosed): diff --git a/internal/ingest/publisher_test.go b/internal/ingest/publisher_test.go new file mode 100644 index 0000000..384e68f --- /dev/null +++ b/internal/ingest/publisher_test.go @@ -0,0 +1,32 @@ +package ingest + +import ( + "errors" + "strings" + "testing" + + "github.com/emersion/go-smtp" + + "codeberg.org/squidd/parsimony/internal/store" +) + +func TestANonPublisherIsRefusedPermanentlyByANewsletter(t *testing.T) { + st := newStack(t, nil) + l, _ := st.store.FindList("dev@example.org") + st.store.SetPolicies(l.ID, store.PostPublishers, store.JoinOpen, store.ArchivePrivate) + res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org"}) + var se *smtp.SMTPError + if !errors.As(res["dev@example.org"], &se) || se.Code != 550 || + se.EnhancedCode != (smtp.EnhancedCode{5, 7, 1}) || !strings.HasSuffix(se.Message, "only the publishers of this list may post") { + t.Fatalf("want 550 5.7.1, got %v", res["dev@example.org"]) + } + if st.queued(t) != 0 { + t.Fatal("nothing may be queued") + } + + st.store.SetPublisher(l.ID, "alice@example.com", true) + if res := deliver(t, st.addr, "alice@example.com", []string{"dev@example.org"}); res["dev@example.org"] != nil { + t.Fatalf("a publisher posts: %v", res["dev@example.org"]) + } + st.sent(t, 2) +} diff --git a/internal/queue/pacer.go b/internal/queue/pacer.go new file mode 100644 index 0000000..0c06221 --- /dev/null +++ b/internal/queue/pacer.go @@ -0,0 +1,81 @@ +package queue + +import ( + "context" + "math" + "sync" + "time" +) + +// maxInterval caps the spacing of a very low rate, so the arithmetic on time never overflows. +const maxInterval = 365 * 24 * time.Hour + +// pacer spaces the start of sends: at most one every interval, with no burst. Relays such as +// Amazon SES refuse mail sent faster than their rate. +type pacer struct { + interval time.Duration + now func() time.Time + sleep func(ctx context.Context, d time.Duration) error + + mu sync.Mutex + next time.Time // the first free slot +} + +// newPacer returns a pacer for perSecond sends per second. It returns nil, which never waits, when +// perSecond is not a positive finite number: the rate is unlimited. +func newPacer(perSecond float64, now func() time.Time, sleep func(ctx context.Context, d time.Duration) error) *pacer { + if !(perSecond > 0) || math.IsInf(perSecond, 0) { + return nil + } + interval := maxInterval + if f := float64(time.Second) / perSecond; f < float64(maxInterval) { + interval = time.Duration(f) + } + return &pacer{interval: interval, now: now, sleep: sleep} +} + +// Wait blocks until the caller's slot. Each caller reserves the next free slot under the lock and +// sleeps outside it. A cancelled context returns its error at once; a slot reserved by a caller +// that gives up is released when no later caller has reserved after it. +func (p *pacer) Wait(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + if p == nil { + return nil + } + p.mu.Lock() + now := p.now() + slot := p.next + if slot.Before(now) { + slot = now + } + p.next = slot.Add(p.interval) + p.mu.Unlock() + + d := slot.Sub(now) + if d <= 0 { + return nil + } + if err := p.sleep(ctx, d); err != nil { + p.mu.Lock() + if p.next.Equal(slot.Add(p.interval)) { + p.next = slot + } + p.mu.Unlock() + return err + } + return nil +} + +// sleepCtx sleeps for d, or until ctx is done. +func sleepCtx(ctx context.Context, d time.Duration) error { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-t.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} diff --git a/internal/queue/pacer_test.go b/internal/queue/pacer_test.go new file mode 100644 index 0000000..4d4ff8b --- /dev/null +++ b/internal/queue/pacer_test.go @@ -0,0 +1,293 @@ +package queue + +import ( + "context" + "errors" + "math" + "sort" + "sync" + "testing" + "time" +) + +// fakeTime is a clock and a sleep that never really sleeps. With advance set, a sleep moves the clock +// (a single caller); otherwise the clock stands still and the durations asked for are recorded +// (many callers reserving slots at the same instant). +type fakeTime struct { + mu sync.Mutex + t time.Time + advance bool + slept []time.Duration + err error // returned by sleep when set +} + +func (f *fakeTime) now() time.Time { + f.mu.Lock() + defer f.mu.Unlock() + return f.t +} + +func (f *fakeTime) move(d time.Duration) { + f.mu.Lock() + f.t = f.t.Add(d) + f.mu.Unlock() +} + +func (f *fakeTime) sleep(ctx context.Context, d time.Duration) error { + f.mu.Lock() + defer f.mu.Unlock() + f.slept = append(f.slept, d) + if f.err != nil { + return f.err + } + if f.advance { + f.t = f.t.Add(d) + } + return nil +} + +func (f *fakeTime) durations() []time.Duration { + f.mu.Lock() + defer f.mu.Unlock() + d := append([]time.Duration(nil), f.slept...) + sort.Slice(d, func(i, j int) bool { return d[i] < d[j] }) + return d +} + +func TestPacerSpacesSendsByTheInterval(t *testing.T) { + f := &fakeTime{t: t0, advance: true} + p := newPacer(10, f.now, f.sleep) // one slot every 100 ms + for i := 0; i < 4; i++ { + if err := p.Wait(context.Background()); err != nil { + t.Fatal(err) + } + if want := t0.Add(time.Duration(i) * 100 * time.Millisecond); !f.now().Equal(want) { + t.Fatalf("send %d starts at +%v, want +%v", i, f.now().Sub(t0), want.Sub(t0)) + } + } +} + +func TestPacerFirstSendDoesNotWait(t *testing.T) { + f := &fakeTime{t: t0, advance: true} + p := newPacer(1, f.now, f.sleep) + p.Wait(context.Background()) + if len(f.slept) != 0 { + t.Fatalf("the first send waited: %v", f.slept) + } +} + +func TestPacerAllowsNoBurstAfterAnIdlePeriod(t *testing.T) { + f := &fakeTime{t: t0, advance: true} + p := newPacer(10, f.now, f.sleep) + p.Wait(context.Background()) + f.move(10 * time.Second) // idle: slots do not pile up + start := f.now() + p.Wait(context.Background()) + p.Wait(context.Background()) + if got := f.now().Sub(start); got != 100*time.Millisecond { + t.Fatalf("two sends after an idle period took %v, want exactly one interval", got) + } +} + +func TestPacerSleepsOnlyWhatIsLeftOfTheInterval(t *testing.T) { + f := &fakeTime{t: t0, advance: true} + p := newPacer(10, f.now, f.sleep) + p.Wait(context.Background()) + f.move(30 * time.Millisecond) // the send itself took 30 ms + p.Wait(context.Background()) + if len(f.slept) != 1 || f.slept[0] != 70*time.Millisecond { + t.Fatalf("slept %v, want [70ms]", f.slept) + } +} + +func TestPacerGivesConcurrentCallersDistinctSlots(t *testing.T) { + const n = 16 + f := &fakeTime{t: t0} // the clock stands still: every caller reserves at the same instant + p := newPacer(4, f.now, f.sleep) + var wg sync.WaitGroup + for i := 0; i < n; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if err := p.Wait(context.Background()); err != nil { + t.Error(err) + } + }() + } + wg.Wait() + got := f.durations() // the first caller does not sleep, so n-1 sleeps: 250ms, 500ms... + if len(got) != n-1 { + t.Fatalf("%d sleeps, want %d: %v", len(got), n-1, got) + } + for i, d := range got { + if want := time.Duration(i+1) * 250 * time.Millisecond; d != want { + t.Fatalf("sleeps %v: slot %d is %v, want %v", got, i+1, d, want) + } + } +} + +func TestPacerCancelledBeforeWaitingReturnsAtOnceAndTakesNoSlot(t *testing.T) { + f := &fakeTime{t: t0} + p := newPacer(1, f.now, f.sleep) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := p.Wait(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v", err) + } + if len(f.slept) != 0 { + t.Fatalf("slept %v", f.slept) + } + // No slot was used: the next caller goes first, without waiting. + if err := p.Wait(context.Background()); err != nil || len(f.slept) != 0 { + t.Fatalf("err=%v slept=%v", err, f.slept) + } +} + +func TestPacerCancelledWhileWaitingGivesTheSlotBack(t *testing.T) { + f := &fakeTime{t: t0} + p := newPacer(10, f.now, f.sleep) + p.Wait(context.Background()) // takes slot 0; the next free slot is +100ms + f.err = context.Canceled + if err := p.Wait(context.Background()); !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v", err) + } + f.err = nil + f.slept = nil + p.Wait(context.Background()) + if len(f.slept) != 1 || f.slept[0] != 100*time.Millisecond { + t.Fatalf("after a cancelled wait the next caller sleeps %v, want [100ms]: the slot was not given back", f.slept) + } +} + +func TestPacerCancelledCallerInTheMiddleNeverBlocksTheOthers(t *testing.T) { + f := &fakeTime{t: t0} + p := newPacer(10, f.now, f.sleep) + p.Wait(context.Background()) + // B and C reserve +100ms and +200ms; B is cancelled and C still gets out at +200ms at most. + var mu sync.Mutex + first := true + f2 := func(ctx context.Context, d time.Duration) error { + if first { + first = false + return context.Canceled + } + mu.Lock() + defer mu.Unlock() + f.slept = append(f.slept, d) + return nil + } + p.sleep = f2 + if err := p.Wait(context.Background()); err == nil { + t.Fatal("B should fail") + } + if err := p.Wait(context.Background()); err != nil { + t.Fatal(err) + } + for _, d := range f.slept { + if d > 200*time.Millisecond { + t.Fatalf("slept %v: later than the slot it reserved", d) + } + } +} + +func TestNoPacerForARateOfZeroOrLess(t *testing.T) { + for _, r := range []float64{0, -1, math.NaN(), math.Inf(-1), math.Inf(1)} { + f := &fakeTime{t: t0} + p := newPacer(r, f.now, f.sleep) + for i := 0; i < 5; i++ { + if err := p.Wait(context.Background()); err != nil { + t.Fatalf("rate %v: %v", r, err) + } + } + if len(f.slept) != 0 { + t.Fatalf("rate %v: slept %v", r, f.slept) + } + } +} + +func TestNoPacerStillStopsForACancelledContext(t *testing.T) { + var p *pacer + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := p.Wait(ctx); err == nil { + t.Fatal("a cancelled context must not pass") + } +} + +func TestPacerAtAVeryLowRateDoesNotOverflow(t *testing.T) { + f := &fakeTime{t: t0, advance: true} + p := newPacer(1e-300, f.now, f.sleep) + for i := 0; i < 3; i++ { + p.Wait(context.Background()) + } + if len(f.slept) != 2 { + t.Fatalf("slept %v", f.slept) + } + for _, d := range f.slept { + if d <= 0 || d > maxInterval { + t.Fatalf("sleep %v out of range (0, %v]", d, maxInterval) + } + } +} + +func TestPacerAtAVeryHighRateNeverWaits(t *testing.T) { + for _, r := range []float64{1e10, 1e12, 1e300, math.MaxFloat64} { + f := &fakeTime{t: t0} + p := newPacer(r, f.now, f.sleep) + for i := 0; i < 100; i++ { + if err := p.Wait(context.Background()); err != nil { + t.Fatal(err) + } + } + if len(f.slept) != 0 { + t.Fatalf("rate %v: slept %v", r, f.slept) + } + } +} + +func TestSleepCtxStopsWhenTheContextIsCancelled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + start := time.Now() + if err := sleepCtx(ctx, time.Hour); !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v", err) + } + if time.Since(start) > time.Second { + t.Fatal("did not return at once") + } + if err := sleepCtx(context.Background(), time.Millisecond); err != nil { + t.Fatal(err) + } +} + +func TestPacerKeepsTheSlotsOfLaterCallersWhenAnEarlierOneGivesUp(t *testing.T) { + f := &fakeTime{t: t0} + release := make(chan struct{}) + p := newPacer(10, f.now, nil) + p.sleep = func(ctx context.Context, d time.Duration) error { + f.sleep(ctx, d) + if d == 100*time.Millisecond { + <-release // B waits until C has reserved its slot, then gives up + return context.Canceled + } + return nil + } + p.Wait(context.Background()) // A: slot 0 + done := make(chan error) + go func() { done <- p.Wait(context.Background()) }() // B: slot +100ms + for len(f.durations()) < 1 { + time.Sleep(time.Millisecond) + } + p.Wait(context.Background()) // C: slot +200ms + close(release) + if err := <-done; err == nil { + t.Fatal("B should have given up") + } + f.mu.Lock() + f.slept = nil + f.mu.Unlock() + p.Wait(context.Background()) // D must come after C, not collide with it + if got := f.durations(); len(got) != 1 || got[0] != 300*time.Millisecond { + t.Fatalf("D sleeps %v, want [300ms]: it took a slot another caller holds", got) + } +} diff --git a/internal/queue/posts_test.go b/internal/queue/posts_test.go new file mode 100644 index 0000000..3736527 --- /dev/null +++ b/internal/queue/posts_test.go @@ -0,0 +1,114 @@ +package queue + +import ( + "context" + "errors" + "testing" + + "github.com/emersion/go-smtp" + + "codeberg.org/squidd/parsimony/internal/outbound" + "codeberg.org/squidd/parsimony/internal/store" +) + +// post queues a post of n recipients (a@x.com, b@x.com...) to a new list and returns the list. +func (e *env) post(t *testing.T, n int) store.List { + t.Helper() + l, err := e.s.CreateList("dev", "example.org", "Dev") + if err != nil { + t.Fatal(err) + } + var items []store.NewOutbound + for i := 0; i < n; i++ { + items = append(items, store.NewOutbound{From: "bounce@x.org", Rcpt: string(rune('a'+i)) + "@x.com"}) + } + if err := e.s.EnqueuePost(l.ID, "mid:", []byte("Message-ID: \r\nSubject: hi\r\n\r\nbody\r\n"), items); err != nil { + t.Fatal(err) + } + return l +} + +func (e *env) onlyPost(t *testing.T, l store.List) store.Post { + t.Helper() + posts, err := e.s.ListPosts(l.ID, 0) + if err != nil || len(posts) != 1 { + t.Fatalf("posts: %+v %v", posts, err) + } + return posts[0] +} + +func TestTheWorkerCountsEachOutcomeOfAPost(t *testing.T) { + e := setup(t) + l := e.post(t, 4) + e.snd.fn = func(_ context.Context, _ string, to []string) error { + switch to[0] { + case "b@x.com": + return errors.New("connection refused") + case "c@x.com": + return &smtp.SMTPError{Code: 554, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "policy"} + case "d@x.com": + return outbound.AtRcpt(&smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"}) + } + return nil + } + st, err := e.w.RunOnce(context.Background()) + if err != nil || st.Sent != 1 || st.Retried != 1 || st.Failed != 1 || st.Dropped != 1 { + t.Fatalf("run: %+v %v", st, err) + } + p := e.onlyPost(t, l) + if p.Recipients != 4 || p.Sent != 1 || p.Failed != 1 || p.Dropped != 1 || p.Pending != 1 { + t.Fatalf("post: %+v", p) + } + // The retried delivery goes out later. + e.snd.fn = nil + e.clk.set(t0.Add(Backoff(1))) + e.w.RunOnce(context.Background()) + if p := e.onlyPost(t, l); p.Sent != 2 || p.Pending != 0 { + t.Fatalf("post: %+v", p) + } +} + +func TestGivingUpAfterMaxAgeCountsAsFailed(t *testing.T) { + e := setup(t) + l := e.post(t, 1) + e.snd.fn = func(context.Context, string, []string) error { return errors.New("down") } + e.clk.set(t0.Add(MaxAge + 1)) + if st, _ := e.w.RunOnce(context.Background()); st.Failed != 1 { + t.Fatalf("stats: %+v", st) + } + if p := e.onlyPost(t, l); p.Failed != 1 || p.Pending != 0 { + t.Fatalf("post: %+v", p) + } +} + +func TestAMissingBodyCountsAsDropped(t *testing.T) { + e := setup(t) + e.w.readBody = func(int64) ([]byte, error) { return nil, store.ErrNotFound } + l := e.post(t, 2) + e.w.RunOnce(context.Background()) + if p := e.onlyPost(t, l); p.Dropped != 2 || p.Sent != 0 || p.Pending != 0 { + t.Fatalf("post: %+v", p) + } +} + +func TestDiscussionMailIsSentBeforeABigNewsletterQueuedEarlier(t *testing.T) { + e := setup(t) + e.w.Concurrency = 1 + var news []store.NewOutbound + for i := 0; i < 120; i++ { + news = append(news, store.NewOutbound{From: "b@x.org", Rcpt: "reader@x.com", Priority: 1}) + } + if err := e.s.Enqueue([]byte("Subject: newsletter\r\n\r\nx"), news); err != nil { + t.Fatal(err) + } + e.clk.set(t0.Add(1)) + e.s.Enqueue([]byte("Subject: talk\r\n\r\nx"), []store.NewOutbound{ + {From: "b@x.org", Rcpt: "first@x.com"}, {From: "b@x.org", Rcpt: "second@x.com"}, + }) + if _, err := e.w.RunOnce(context.Background()); err != nil { + t.Fatal(err) + } + if e.snd.sent[0].To[0] != "first@x.com" || e.snd.sent[1].To[0] != "second@x.com" || e.snd.sent[2].To[0] != "reader@x.com" { + t.Fatalf("order: %v %v %v", e.snd.sent[0].To, e.snd.sent[1].To, e.snd.sent[2].To) + } +} diff --git a/internal/queue/queue.go b/internal/queue/queue.go index 9976a9c..de5a634 100644 --- a/internal/queue/queue.go +++ b/internal/queue/queue.go @@ -60,6 +60,15 @@ type Worker struct { // Sign, when set, signs the final message (DKIM). On error the message is sent unsigned: // not delivering is worse than a missed signature. Sign func(msg []byte) ([]byte, error) + // Rate is the most sends started per second, for relays that throttle (Amazon SES). Zero or + // negative means unlimited. It is read once, at the first send. + Rate float64 + + pacerOnce sync.Once + pacer *pacer + // Hooks for tests; nil means the real clock and sleep. + pacerNow func() time.Time + pacerSleep func(ctx context.Context, d time.Duration) error wakeOnce sync.Once wake chan struct{} @@ -219,7 +228,7 @@ func (w *Worker) RunOnce(ctx context.Context) (Stats, error) { func (w *Worker) deliver(ctx context.Context, cache *bodies, it store.OutboxItem) Stats { data, err := cache.get(it.MsgID) if errors.Is(err, store.ErrNotFound) { - w.complete(it) + w.drop(it) return Stats{Dropped: 1} } if err != nil { @@ -240,6 +249,10 @@ func (w *Worker) deliver(ctx context.Context, cache *bodies, it store.OutboxItem if timeout <= 0 { timeout = defaultTimeout } + // Wait for the slot before starting the timeout, so waiting does not eat into the send. + if err := w.pace().Wait(ctx); err != nil { + return Stats{} // shutting down: the delivery stays queued, unchanged + } sctx, cancel := context.WithTimeout(ctx, timeout) err = w.Sender.Send(sctx, it.From, []string{it.Rcpt}, msg) cancel() @@ -259,7 +272,7 @@ func (w *Worker) deliver(ctx context.Context, cache *bodies, it store.OutboxItem } } log.Printf("queue: drop %s: %v", it.Rcpt, err) - w.complete(it) + w.drop(it) return out case outbound.IsPermanent(err): // The relay refused the message itself (policy, size, line length...): nobody gets it, so @@ -277,6 +290,21 @@ func (w *Worker) deliver(ctx context.Context, cache *bodies, it store.OutboxItem return Stats{Retried: 1} } +// pace returns the pacer in front of every send; nil (never waits) when Rate is not set. +func (w *Worker) pace() *pacer { + w.pacerOnce.Do(func() { + now, sleep := w.pacerNow, w.pacerSleep + if now == nil { + now = time.Now + } + if sleep == nil { + sleep = sleepCtx + } + w.pacer = newPacer(w.Rate, now, sleep) + }) + return w.pacer +} + // giveUp marks a delivery as failed for good and logs it. func (w *Worker) giveUp(it store.OutboxItem, reason string) Stats { log.Printf("queue: %s: %s", it.Rcpt, reason) @@ -292,3 +320,10 @@ func (w *Worker) complete(it store.OutboxItem) { log.Printf("queue: remove %d: %v", it.ID, err) } } + +// drop removes a delivery that will never be made; a delivery of a post counts as dropped. +func (w *Worker) drop(it store.OutboxItem) { + if err := w.Store.DropOutbox(it.ID); err != nil && !errors.Is(err, store.ErrNotFound) { + log.Printf("queue: drop %d: %v", it.ID, err) + } +} diff --git a/internal/queue/queue_test.go b/internal/queue/queue_test.go index 7e2dbd2..24ae823 100644 --- a/internal/queue/queue_test.go +++ b/internal/queue/queue_test.go @@ -668,3 +668,91 @@ func TestAMissingBodyIsDroppedForEveryRecipientWithOneRead(t *testing.T) { t.Fatalf("%d reads", n) } } + +func TestRateSpacesTheSendsOfAWorker(t *testing.T) { + e := setup(t) + e.enqueue(t, 5) + e.w.Rate = 2 // one send every 500 ms + f := &fakeTime{t: t0} + e.w.pacerNow, e.w.pacerSleep = f.now, f.sleep + st, _ := e.w.RunOnce(context.Background()) + if st.Sent != 5 { + t.Fatalf("%+v", st) + } + got := f.durations() // 5 deliveries: slots at 0, 1, 2, 3 and 4 intervals + want := []time.Duration{500 * time.Millisecond, time.Second, 1500 * time.Millisecond, 2 * time.Second} + if len(got) != len(want) { + t.Fatalf("sleeps %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("sleeps %v, want %v", got, want) + } + } +} + +func TestRateOfZeroNeverWaits(t *testing.T) { + e := setup(t) + e.enqueue(t, 5) + f := &fakeTime{t: t0} + e.w.pacerNow, e.w.pacerSleep = f.now, f.sleep + e.w.RunOnce(context.Background()) + if len(f.slept) != 0 || e.snd.count() != 5 { + t.Fatalf("slept %v, sent %d", f.slept, e.snd.count()) + } +} + +func TestEverySendAttemptIsPacedEvenARetry(t *testing.T) { + e := setup(t) + e.enqueue(t, 1) + e.w.Rate = 2 + f := &fakeTime{t: t0} + e.w.pacerNow, e.w.pacerSleep = f.now, f.sleep + e.snd.fn = func(context.Context, string, []string) error { return errors.New("try later") } + e.w.RunOnce(context.Background()) + e.clk.set(t0.Add(time.Hour)) + e.w.RunOnce(context.Background()) + if e.snd.calls != 2 || len(f.slept) != 1 { + t.Fatalf("calls=%d sleeps=%v: the second attempt shares the pacer with the first", e.snd.calls, f.slept) + } +} + +func TestPacingIsCancelledWithoutRecordingAnAttempt(t *testing.T) { + e := setup(t) + e.enqueue(t, 2) + e.w.Rate = 1 + e.w.Concurrency = 1 + ctx, cancel := context.WithCancel(context.Background()) + f := &fakeTime{t: t0} + e.w.pacerNow = f.now + e.w.pacerSleep = func(ctx context.Context, d time.Duration) error { + cancel() // shutdown arrives while the delivery waits for its slot + return ctx.Err() + } + st, _ := e.w.RunOnce(ctx) + if st != (Stats{Sent: 1}) { + t.Fatalf("stats %+v: the delivery that did not get its slot is not counted", st) + } + list, _ := e.s.ListOutbox(10) + if len(list) != 1 || list[0].Attempts != 0 || list[0].LastError != "" { + t.Fatalf("a shutdown while waiting is not a delivery failure: %+v", list) + } + if e.snd.calls != 1 { + t.Fatalf("calls = %d: only the first delivery, which did not wait, was sent", e.snd.calls) + } +} + +func TestRateHoldsInRealTime(t *testing.T) { + e := setup(t) + e.enqueue(t, 20) + e.w.Rate = 200 // 5 ms apart: 19 intervals take 95 ms + start := time.Now() + st, _ := e.w.RunOnce(context.Background()) + took := time.Since(start) + if st.Sent != 20 { + t.Fatalf("%+v", st) + } + if took < 90*time.Millisecond || took > time.Second { + t.Fatalf("20 sends at 200/s took %v, want about 95 ms", took) + } +} diff --git a/internal/router/commands.go b/internal/router/commands.go index bd8ab3d..b8c621a 100644 --- a/internal/router/commands.go +++ b/internal/router/commands.go @@ -94,6 +94,24 @@ func (r *Router) join(l store.List, sender string) error { return r.askConfirmation(l, sender, store.ActionJoin) } +// RequestJoin does what the +join command does for addr, for a caller that is not an LMTP session +// (the public subscribe form): there is no envelope sender to check, so the caller must have +// validated addr. It returns nil whenever nothing should be revealed: an unknown or closed list, an +// active member, a request the store rate limits. It returns an error only for internal failures. +func (r *Router) RequestJoin(listAddress, addr string) error { + l, err := r.store.FindList(listAddress) + if errors.Is(err, store.ErrNotFound) { + return nil + } + if err != nil { + return err + } + if err := r.join(l, addr); err != nil && !errors.Is(err, ErrJoinClosed) { + return err + } + return nil +} + func (r *Router) leave(l store.List, sender string) error { if ok, err := r.store.IsMember(l.ID, sender); err != nil || !ok { return err diff --git a/internal/router/m2_test.go b/internal/router/m2_test.go index ab9d060..92c96c7 100644 --- a/internal/router/m2_test.go +++ b/internal/router/m2_test.go @@ -628,3 +628,76 @@ func TestDroppingARCHeadersKeepsSimilarlyNamedOnes(t *testing.T) { t.Errorf("only the ARC- prefix is dropped: %v", h) } } + +func TestRequestJoinQueuesTheConfirmationMail(t *testing.T) { + e := newEnv(t) + if err := e.r.RequestJoin("dev@example.org", "carol@x.com"); err != nil { + t.Fatal(err) + } + e.flush() + m := lastMail(t, e.out) + if len(m.To) != 1 || m.To[0] != "carol@x.com" { + t.Fatalf("confirmation must go to the requester: %+v", m.To) + } + if ok, _ := e.s.IsMember(e.list.ID, "carol@x.com"); ok { + t.Fatal("not a member before confirming") + } + // The mail is the usual one: replying to it confirms. + if _, err := e.command("carol@x.com", "dev+confirm-"+token(t, m)+"@example.org", nil); err != nil { + t.Fatal(err) + } + if ok, _ := e.s.IsMember(e.list.ID, "carol@x.com"); !ok { + t.Fatal("carol must be a member after confirming") + } +} + +// Nothing may tell the caller whether the list exists, is closed, or the address is known. +func TestRequestJoinRevealsNothing(t *testing.T) { + e := newEnv(t) + closed, _ := e.s.CreateList("closed", "example.org", "Closed") + e.s.SetPolicies(closed.ID, store.PostMembers, store.JoinClosed, store.ArchivePrivate) + for _, c := range []struct{ name, list, addr string }{ + {"unknown list", "nope@example.org", "carol@x.com"}, + {"closed list", "closed@example.org", "carol@x.com"}, + {"active member", "dev@example.org", "alice@example.com"}, + } { + if err := e.r.RequestJoin(c.list, c.addr); err != nil { + t.Errorf("%s: want nil, got %v", c.name, err) + } + } + e.flush() + if len(e.out.Sent) != 0 { + t.Fatalf("no mail for these cases, sent %d", len(e.out.Sent)) + } + // Asking twice is rate limited by the store: still nil, still one mail. + e.r.RequestJoin("dev@example.org", "carol@x.com") + if err := e.r.RequestJoin("dev@example.org", "carol@x.com"); err != nil { + t.Fatalf("rate limited: want nil, got %v", err) + } + e.flush() + if len(e.out.Sent) != 1 { + t.Fatalf("one mail expected, sent %d", len(e.out.Sent)) + } +} + +func TestRequestJoinLetsABounceDisabledMemberRejoin(t *testing.T) { + e := newEnv(t) + for i := 0; i < store.BounceThreshold; i++ { + e.s.RecordHardBounce(e.alice, store.BounceThreshold) + } + if err := e.r.RequestJoin("dev@example.org", "alice@example.com"); err != nil { + t.Fatal(err) + } + e.flush() + if m := lastMail(t, e.out); len(m.To) != 1 || m.To[0] != "alice@example.com" { + t.Fatalf("to = %v", m.To) + } +} + +func TestRequestJoinReportsAnInternalFailure(t *testing.T) { + e := newEnv(t) + e.s.Close() + if err := e.r.RequestJoin("dev@example.org", "carol@x.com"); err == nil { + t.Fatal("a database failure must be reported") + } +} diff --git a/internal/router/priority_test.go b/internal/router/priority_test.go new file mode 100644 index 0000000..ceab018 --- /dev/null +++ b/internal/router/priority_test.go @@ -0,0 +1,113 @@ +package router + +import ( + "testing" + + "codeberg.org/squidd/parsimony/internal/store" +) + +// priorityEnv is newEnv with the publishers post policy, for the delivery priority tests. +func priorityEnv(t *testing.T) *env { + t.Helper() + e := newEnv(t) + if err := e.s.SetPolicies(e.list.ID, store.PostPublishers, store.JoinOpen, store.ArchivePrivate); err != nil { + t.Fatal(err) + } + l, _ := e.s.GetList(e.list.ID) + e.list = l + return e +} + +func duePriorities(t *testing.T, s *store.Store) []int { + t.Helper() + due, err := s.DueOutbox(100) + if err != nil { + t.Fatal(err) + } + var out []int + for _, d := range due { + out = append(out, d.Priority) + } + return out +} + +func allEqual(ps []int, want int, n int) bool { + if len(ps) != n { + return false + } + for _, p := range ps { + if p != want { + return false + } + } + return true +} + +const samplePost = "From: alice@example.com\r\nSubject: hi\r\nMessage-ID: \r\n\r\nbody\r\n" + +func TestANewsletterIsDeliveredWithPriorityOne(t *testing.T) { + e := priorityEnv(t) + res, err := e.r.prepare(e.list, []byte(samplePost)) + if err != nil { + t.Fatal(err) + } + if err := e.r.Deliver(res); err != nil { + t.Fatal(err) + } + if ps := duePriorities(t, e.s); !allEqual(ps, 1, 2) { + t.Fatalf("priorities: %v", ps) + } +} + +func TestADiscussionPostIsDeliveredWithPriorityZero(t *testing.T) { + e := newEnv(t) + if err := e.post(t, samplePost); err != nil { + t.Fatal(err) + } + if ps := duePriorities(t, e.s); !allEqual(ps, 0, 2) { + t.Fatalf("priorities: %v", ps) + } +} + +func TestAReleasedNewsletterHasPriorityOne(t *testing.T) { + e := priorityEnv(t) + id, _ := e.s.HoldMessage(e.list.ID, "alice@example.com", []byte(samplePost), "r") + if err := e.r.Release(id); err != nil { + t.Fatal(err) + } + if ps := duePriorities(t, e.s); !allEqual(ps, 1, 2) { + t.Fatalf("priorities: %v", ps) + } +} + +func TestAReleasedDiscussionPostHasPriorityZero(t *testing.T) { + e := newEnv(t) + id, _ := e.s.HoldMessage(e.list.ID, "alice@example.com", []byte(samplePost), "r") + if err := e.r.Release(id); err != nil { + t.Fatal(err) + } + if ps := duePriorities(t, e.s); !allEqual(ps, 0, 2) { + t.Fatalf("priorities: %v", ps) + } +} + +func TestConfirmationMailOfANewsletterHasPriorityZero(t *testing.T) { + e := priorityEnv(t) + if handled, err := e.r.Command("carol@x.com", "dev+join@example.org", nil); !handled || err != nil { + t.Fatalf("join: %v %v", handled, err) + } + if ps := duePriorities(t, e.s); !allEqual(ps, 0, 1) { + t.Fatalf("priorities: %v", ps) + } +} + +func TestAPostIsRecordedWithItsRecipients(t *testing.T) { + e := newEnv(t) + if err := e.post(t, samplePost); err != nil { + t.Fatal(err) + } + posts, err := e.s.ListPosts(e.list.ID, 0) + if err != nil || len(posts) != 1 || posts[0].Recipients != 2 || posts[0].MessageID != "" || posts[0].Subject == "" { + t.Fatalf("posts: %+v %v", posts, err) + } +} diff --git a/internal/router/publisher_test.go b/internal/router/publisher_test.go new file mode 100644 index 0000000..c80f3db --- /dev/null +++ b/internal/router/publisher_test.go @@ -0,0 +1,143 @@ +package router + +import ( + "errors" + "strings" + "testing" + + "codeberg.org/squidd/parsimony/internal/store" +) + +// newsletterEnv is the test list with the publishers post policy and alice as its only publisher. +func newsletterEnv(t *testing.T) *env { + t.Helper() + e := newEnv(t) + if err := e.s.SetPolicies(e.list.ID, store.PostPublishers, store.JoinOpen, store.ArchivePrivate); err != nil { + t.Fatal(err) + } + if err := e.s.SetPublisher(e.list.ID, "alice@example.com", true); err != nil { + t.Fatal(err) + } + return e +} + +// nothingKept checks that a refused post left no trace: not held, not archived, not queued. +func nothingKept(t *testing.T, e *env) { + t.Helper() + if held, _ := e.s.ListHeld(e.list.ID); len(held) != 0 { + t.Fatalf("held: %+v", held) + } + if msgs, _, _ := e.s.ListMessages(e.list.ID, 10, 0); len(msgs) != 0 { + t.Fatalf("archived: %+v", msgs) + } + if st, _ := e.s.OutboxStats(); st.Pending != 0 { + t.Fatalf("queued: %+v", st) + } +} + +func TestAPublisherPostsToANewsletter(t *testing.T) { + e := newsletterEnv(t) + res, err := e.r.Route("Alice@Example.com", "dev@example.org", postFrom("Alice ")) + if err != nil { + t.Fatal(err) + } + if err := e.send(res); err != nil { + t.Fatal(err) + } + if len(e.out.Sent) != 2 { + t.Fatalf("every member receives it: sent %d", len(e.out.Sent)) + } +} + +func TestAPlainMemberMayNotPostToANewsletter(t *testing.T) { + e := newsletterEnv(t) + _, err := e.r.Route("bob@example.com", "dev@example.org", postFrom("bob@example.com")) + if !errors.Is(err, ErrNotPublisher) { + t.Fatalf("want ErrNotPublisher, got %v", err) + } + nothingKept(t, e) +} + +func TestAStrangerMayNotPostToANewsletter(t *testing.T) { + e := newsletterEnv(t) + for _, from := range []string{"mallory@evil.com", "alice@example.com"} { + _, err := e.r.Route("mallory@evil.com", "dev@example.org", postFrom(from)) + if !errors.Is(err, ErrNotPublisher) { + t.Fatalf("From %s: want ErrNotPublisher, got %v", from, err) + } + } + nothingKept(t, e) +} + +func TestAPublisherDisabledByBouncesStillPosts(t *testing.T) { + e := newsletterEnv(t) + for i := 0; i < store.BounceThreshold; i++ { + e.s.RecordHardBounce(e.alice, store.BounceThreshold) + } + res, err := e.r.Route("alice@example.com", "dev@example.org", postFrom("alice@example.com")) + if err != nil { + t.Fatal(err) + } + if len(res.Recipients) != 1 || res.Recipients[0].Address != "bob@example.com" { + t.Fatalf("a disabled publisher posts but does not receive: %+v", res.Recipients) + } +} + +func TestAPublisherWithAnotherFromHeaderIsHeld(t *testing.T) { + e := newsletterEnv(t) + _, err := e.r.Route("alice@example.com", "dev@example.org", postFrom("Bob ")) + if !errors.Is(err, ErrHeld) { + t.Fatalf("want ErrHeld, got %v", err) + } + held, _ := e.s.ListHeld(e.list.ID) + if len(held) != 1 || !strings.Contains(held[0].Reason, "From header does not match") { + t.Fatalf("held: %+v", held) + } + if st, _ := e.s.OutboxStats(); st.Pending != 0 { + t.Fatal("nothing may be queued") + } +} + +func TestAReleasedPublisherPostIsDelivered(t *testing.T) { + e := newsletterEnv(t) + e.r.Route("alice@example.com", "dev@example.org", postFrom("Bob ")) + held, _ := e.s.ListHeld(e.list.ID) + if err := e.release(held[0].ID); err != nil { + t.Fatal(err) + } + if len(e.out.Sent) != 2 { + t.Fatalf("the admin approved it: sent %d", len(e.out.Sent)) + } + if held, _ := e.s.ListHeld(e.list.ID); len(held) != 0 { + t.Fatal("released") + } +} + +func TestNewsletterDiscardRulesComeFirst(t *testing.T) { + e := newsletterEnv(t) + for _, sender := range []string{"alice@example.com", "bob@example.com", "stranger@x.com"} { + for _, h := range []string{"List-Id: \r\n", "Auto-Submitted: auto-replied\r\n"} { + if _, err := e.r.Route(sender, "dev@example.org", withHeaders(h)); !errors.Is(err, ErrDiscarded) { + t.Fatalf("%s %q: want ErrDiscarded, got %v", sender, h, err) + } + } + } + nothingKept(t, e) +} + +func TestAPublisherFlagChangesNothingOnOtherPolicies(t *testing.T) { + e := newEnv(t) + // members: a plain member posts, a stranger is refused as a non-member. + if _, err := e.r.Route("bob@example.com", "dev@example.org", postFrom("bob@example.com")); err != nil { + t.Fatalf("members policy: %v", err) + } + if _, err := e.r.Route("x@y.com", "dev@example.org", postFrom("x@y.com")); !errors.Is(err, ErrNotMember) { + t.Fatalf("members policy, stranger: %v", err) + } + // moderated: a publisher is held like everybody else. + e.s.SetPublisher(e.list.ID, "alice@example.com", true) + e.s.SetPolicies(e.list.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate) + if _, err := e.r.Route("alice@example.com", "dev@example.org", postFrom("alice@example.com")); !errors.Is(err, ErrHeld) { + t.Fatalf("moderated policy: %v", err) + } +} diff --git a/internal/router/router.go b/internal/router/router.go index 7e95813..71230ac 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -28,6 +28,8 @@ var ( ErrHeldTooLarge = store.ErrHeldTooLarge // ErrNotMember means the sender may not post to the list. ErrNotMember = errors.New("sender is not a member") + // ErrNotPublisher means the list takes posts from its publishers only and the sender is not one. + ErrNotPublisher = errors.New("sender is not a publisher") // ErrHeld means the message was queued for moderation. It is not a failure. ErrHeld = errors.New("held for moderation") // ErrBadToken means a confirmation or bounce token is invalid, used or expired. @@ -87,12 +89,23 @@ func (r *Router) Route(sender, rcpt string, data []byte) (Result, error) { if l.PostPolicy == store.PostModerated { return Result{}, r.hold(l, sender, data, "the list is moderated") } - ok, err := r.store.IsMember(l.ID, sender) - if err != nil { - return Result{}, err - } - if !ok { - return Result{}, ErrNotMember + if l.PostPolicy == store.PostPublishers { + // A newsletter: only its publishers may post. Everybody else is refused, never held. + ok, err := r.store.IsPublisher(l.ID, sender) + if err != nil { + return Result{}, err + } + if !ok { + return Result{}, ErrNotPublisher + } + } else { + ok, err := r.store.IsMember(l.ID, sender) + if err != nil { + return Result{}, err + } + if !ok { + return Result{}, ErrNotMember + } } // Membership is judged on the envelope sender, which anyone can forge, while the archive, the // rewritten From and Reply-To come from the From header. Only trust a post whose header names @@ -163,8 +176,13 @@ func postID(data []byte) string { return "sha256:" + hex.EncodeToString(sum[:]) } -// outbound builds one queue entry per recipient. +// outbound builds one queue entry per recipient. A newsletter (post policy publishers) is sent +// with priority 1, so that its many deliveries never delay discussion mail. func (r *Router) outbound(res Result) ([]store.NewOutbound, error) { + priority := 0 + if res.List.PostPolicy == store.PostPublishers { + priority = 1 + } items := make([]store.NewOutbound, 0, len(res.Recipients)) for _, rc := range res.Recipients { from := r.bounceAddress(res.List, rc.ID) @@ -172,7 +190,7 @@ func (r *Router) outbound(res Result) ([]store.NewOutbound, error) { return nil, errors.New("cannot create bounce address") } items = append(items, store.NewOutbound{ - From: from, Rcpt: rc.Address, MemberID: rc.ID, Prefix: r.unsubscribeHeaders(res.List, rc.ID), + From: from, Rcpt: rc.Address, MemberID: rc.ID, Prefix: r.unsubscribeHeaders(res.List, rc.ID), Priority: priority, }) } return items, nil diff --git a/internal/store/outbox.go b/internal/store/outbox.go index c72abea..a0aea3b 100644 --- a/internal/store/outbox.go +++ b/internal/store/outbox.go @@ -12,6 +12,7 @@ type NewOutbound struct { Rcpt string MemberID int64 // 0 for system mail Prefix []byte // per-recipient headers, sent before the shared message + Priority int // lower goes first among due deliveries: 0 for discussion and system mail, 1 for a newsletter } // OutboxItem is a queued delivery to one recipient. @@ -28,7 +29,8 @@ type OutboxItem struct { CreatedAt time.Time // Failed deliveries are never retried (the relay refused them for good, or they are too old). // They stay listed for a week so the admin can see what was lost and why. - Failed bool + Failed bool + Priority int } // OutboxStats summarises the queue. @@ -52,7 +54,7 @@ func (s *Store) Enqueue(msg []byte, items []NewOutbound) error { return err } defer tx.Rollback() - if err := enqueueTx(tx, s.Now().Unix(), msg, items); err != nil { + if err := enqueueTx(tx, s.Now().Unix(), 0, msg, items); err != nil { return err } return tx.Commit() @@ -65,6 +67,8 @@ const postMemory = time.Hour // in the same transaction, so a crash never leaves one without the other. If the list already // queued the same postID within postMemory it queues nothing and returns ErrDuplicate: the MTA // delivers the post again when our reply to the first delivery was lost. +// It also records the post (a Post, with its counters) and stamps every delivery with its id; +// a refused duplicate records nothing. func (s *Store) EnqueuePost(listID int64, postID string, msg []byte, items []NewOutbound) error { now := s.Now() tx, err := s.db.Begin() @@ -81,8 +85,12 @@ func (s *Store) EnqueuePost(listID int64, postID string, msg []byte, items []New } return err } + post, err := recordPostTx(tx, now, listID, msg, len(items)) + if err != nil { + return err + } if len(items) > 0 { - if err := enqueueTx(tx, now.Unix(), msg, items); err != nil { + if err := enqueueTx(tx, now.Unix(), post, msg, items); err != nil { return err } } @@ -93,7 +101,7 @@ func (s *Store) EnqueuePost(listID int64, postID string, msg []byte, items []New // its deliveries in one transaction. Of several simultaneous approvals only one succeeds, the // others get ErrNotFound, so a message is never delivered twice. With no recipients the message // is still released. An approval always delivers, even a post already seen, and records postID -// as EnqueuePost does, so a later copy of the same post is refused. +// as EnqueuePost does, so a later copy of the same post is refused. It records the post as EnqueuePost does. func (s *Store) EnqueueFromHeld(heldID int64, postID string, msg []byte, items []NewOutbound) error { tx, err := s.db.Begin() if err != nil { @@ -113,15 +121,20 @@ func (s *Store) EnqueueFromHeld(heldID int64, postID string, msg []byte, items [ ON CONFLICT(list_id, post_id) DO UPDATE SET created_at=excluded.created_at`, listID, postID, s.Now().Unix()); err != nil { return err } + post, err := recordPostTx(tx, s.Now(), listID, msg, len(items)) + if err != nil { + return err + } if len(items) > 0 { - if err := enqueueTx(tx, s.Now().Unix(), msg, items); err != nil { + if err := enqueueTx(tx, s.Now().Unix(), post, msg, items); err != nil { return err } } return tx.Commit() } -func enqueueTx(tx *sql.Tx, now int64, msg []byte, items []NewOutbound) error { +// enqueueTx stores msg and its deliveries; postID is the posts row they count for, 0 for other mail. +func enqueueTx(tx *sql.Tx, now, postID int64, msg []byte, items []NewOutbound) error { res, err := tx.Exec(`INSERT INTO outbox_msgs(data) VALUES(?)`, msg) if err != nil { return err @@ -132,20 +145,20 @@ func enqueueTx(tx *sql.Tx, now int64, msg []byte, items []NewOutbound) error { if prefix == nil { prefix = []byte{} } - if _, err := tx.Exec(`INSERT INTO outbox(msg_id, envelope_from, rcpt, member_id, prefix, next_at, created_at) - VALUES(?,?,?,?,?,?,?)`, msgID, it.From, it.Rcpt, it.MemberID, prefix, now, now); err != nil { + if _, err := tx.Exec(`INSERT INTO outbox(msg_id, envelope_from, rcpt, member_id, prefix, next_at, created_at, post_id, priority) + VALUES(?,?,?,?,?,?,?,?,?)`, msgID, it.From, it.Rcpt, it.MemberID, prefix, now, now, postID, it.Priority); err != nil { return err } } return nil } -const outboxCols = `id, msg_id, envelope_from, rcpt, member_id, prefix, attempts, next_at, last_error, created_at, failed` +const outboxCols = `id, msg_id, envelope_from, rcpt, member_id, prefix, attempts, next_at, last_error, created_at, failed, priority` func scanOutbox(row interface{ Scan(...any) error }) (OutboxItem, error) { var it OutboxItem var next, created, failed int64 - err := row.Scan(&it.ID, &it.MsgID, &it.From, &it.Rcpt, &it.MemberID, &it.Prefix, &it.Attempts, &next, &it.LastError, &created, &failed) + err := row.Scan(&it.ID, &it.MsgID, &it.From, &it.Rcpt, &it.MemberID, &it.Prefix, &it.Attempts, &next, &it.LastError, &created, &failed, &it.Priority) it.NextAt, it.CreatedAt, it.Failed = time.Unix(next, 0), time.Unix(created, 0), failed != 0 return it, err } @@ -167,9 +180,10 @@ func (s *Store) queryOutbox(q string, args ...any) ([]OutboxItem, error) { return out, rows.Err() } -// DueOutbox returns up to limit deliveries whose time has come, longest waiting first. +// DueOutbox returns up to limit deliveries whose time has come: lowest priority number first (so a +// big newsletter never delays discussion mail), then longest waiting first. func (s *Store) DueOutbox(limit int) ([]OutboxItem, error) { - return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox WHERE failed=0 AND next_at<=? ORDER BY next_at, id LIMIT ?`, + return s.queryOutbox(`SELECT `+outboxCols+` FROM outbox WHERE failed=0 AND next_at<=? ORDER BY priority, next_at, id LIMIT ?`, s.Now().Unix(), limit) } @@ -188,16 +202,26 @@ func (s *Store) OutboxData(msgID int64) ([]byte, error) { return b, err } -// CompleteOutbox removes a delivery (sent, dropped or discarded) and the shared -// message once no recipient needs it. -func (s *Store) CompleteOutbox(id int64) error { +// CompleteOutbox removes a delivery the relay accepted, and the shared message once no recipient +// needs it. A delivery of a post counts as sent, in the same transaction. +func (s *Store) CompleteOutbox(id int64) error { return s.removeOutbox(id, "sent") } + +// DropOutbox removes a delivery that will never be made (the mailbox does not exist, or the admin +// discarded it), like CompleteOutbox. A delivery of a post counts as dropped, unless it had +// already failed: it is counted as failed already. +func (s *Store) DropOutbox(id int64) error { return s.removeOutbox(id, "dropped") } + +// removeOutbox deletes a delivery and adds one to counter (a column of posts, never user input) +// of its post, when it has one and it had not failed. +func (s *Store) removeOutbox(id int64, counter string) error { tx, err := s.db.Begin() if err != nil { return err } defer tx.Rollback() - var msgID int64 - err = tx.QueryRow(`SELECT msg_id FROM outbox WHERE id=?`, id).Scan(&msgID) + var msgID, postID int64 + var failed bool + err = tx.QueryRow(`SELECT msg_id, post_id, failed FROM outbox WHERE id=?`, id).Scan(&msgID, &postID, &failed) if errors.Is(err, sql.ErrNoRows) { return ErrNotFound } @@ -210,6 +234,11 @@ func (s *Store) CompleteOutbox(id int64) error { if _, err := tx.Exec(`DELETE FROM outbox_msgs WHERE id=? AND NOT EXISTS (SELECT 1 FROM outbox WHERE msg_id=?)`, msgID, msgID); err != nil { return err } + if postID != 0 && !failed { + if _, err := tx.Exec(`UPDATE posts SET `+counter+`=`+counter+`+1 WHERE id=?`, postID); err != nil { + return err + } + } return tx.Commit() } @@ -232,10 +261,33 @@ func (s *Store) OutboxStats() (OutboxStats, error) { } // FailOutbox gives up on a delivery for good, keeping it listed with the reason. Its time is the -// moment of the failure, from which PurgeFailedOutbox counts. +// moment of the failure, from which PurgeFailedOutbox counts. A delivery of a post counts as failed, +// in the same transaction. func (s *Store) FailOutbox(id int64, reason string) error { - return affected(s.db.Exec(`UPDATE outbox SET failed=1, attempts=attempts+1, last_error=?, next_at=? WHERE id=?`, - reason, s.Now().Unix(), id)) + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + var postID int64 + var failed bool + err = tx.QueryRow(`SELECT post_id, failed FROM outbox WHERE id=?`, id).Scan(&postID, &failed) + if errors.Is(err, sql.ErrNoRows) { + return ErrNotFound + } + if err != nil { + return err + } + if _, err := tx.Exec(`UPDATE outbox SET failed=1, attempts=attempts+1, last_error=?, next_at=? WHERE id=?`, + reason, s.Now().Unix(), id); err != nil { + return err + } + if postID != 0 && !failed { + if _, err := tx.Exec(`UPDATE posts SET failed=failed+1 WHERE id=?`, postID); err != nil { + return err + } + } + return tx.Commit() } // PurgeFailedOutbox forgets failed deliveries older than a week, and the stored messages nobody diff --git a/internal/store/posts.go b/internal/store/posts.go new file mode 100644 index 0000000..e9f3323 --- /dev/null +++ b/internal/store/posts.go @@ -0,0 +1,78 @@ +package store + +import ( + "bytes" + "database/sql" + "net/mail" + "time" +) + +// Post is a post queued to a list, with the outcome of its deliveries so far. +type Post struct { + ID int64 + ListID int64 + MessageID string // the Message-ID header, at most 200 characters + Subject string // decoded, at most 200 characters + CreatedAt time.Time + Recipients int // deliveries queued + Sent int // accepted by the relay + Failed int // given up for good + Dropped int // the mailbox does not exist, or discarded by the admin + Pending int // still in the queue: Recipients - Sent - Failed - Dropped +} + +// postTTL is how long a post and its counters are kept. +const postTTL = 400 * 24 * time.Hour + +// Limits of ListPosts. +const ( + defaultPostsLimit = 50 + maxPostsLimit = 200 +) + +// recordPostTx stores the posts row of a post queued to a list and returns its id. It forgets +// posts older than postTTL on the way, so the table never grows without bound. +func recordPostTx(tx *sql.Tx, now time.Time, listID int64, msg []byte, recipients int) (int64, error) { + if _, err := tx.Exec(`DELETE FROM posts WHERE created_at maxPostsLimit { + limit = maxPostsLimit + } + rows, err := s.db.Query(`SELECT id, list_id, message_id, subject, created_at, recipients, sent, failed, dropped + FROM posts WHERE list_id=? ORDER BY created_at DESC, id DESC LIMIT ?`, listID, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Post + for rows.Next() { + var p Post + var created int64 + if err := rows.Scan(&p.ID, &p.ListID, &p.MessageID, &p.Subject, &created, &p.Recipients, &p.Sent, &p.Failed, &p.Dropped); err != nil { + return nil, err + } + p.CreatedAt = time.Unix(created, 0) + p.Pending = p.Recipients - p.Sent - p.Failed - p.Dropped + out = append(out, p) + } + return out, rows.Err() +} diff --git a/internal/store/posts_test.go b/internal/store/posts_test.go new file mode 100644 index 0000000..24a4960 --- /dev/null +++ b/internal/store/posts_test.go @@ -0,0 +1,424 @@ +package store + +import ( + "errors" + "strings" + "testing" + "time" +) + +const postMsg = "Message-ID: \r\nSubject: =?utf-8?q?Caff=C3=A8?= news\x07\r\n\r\nbody" + +func onlyPost(t *testing.T, s *Store, listID int64) Post { + t.Helper() + posts, err := s.ListPosts(listID, 0) + if err != nil || len(posts) != 1 { + t.Fatalf("posts: %+v %v", posts, err) + } + return posts[0] +} + +func outboxPostIDs(t *testing.T, s *Store) []int64 { + t.Helper() + rows, err := s.db.Query(`SELECT post_id FROM outbox ORDER BY id`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var ids []int64 + for rows.Next() { + var id int64 + rows.Scan(&id) + ids = append(ids, id) + } + return ids +} + +func TestEnqueuePostRecordsThePost(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + l, _ := s.CreateList("dev", "example.org", "Dev") + if err := s.EnqueuePost(l.ID, "mid:", []byte(postMsg), items(3)); err != nil { + t.Fatal(err) + } + p := onlyPost(t, s, l.ID) + if p.ID == 0 || p.ListID != l.ID || p.MessageID != "" || p.Subject != "Caffè news" || !p.CreatedAt.Equal(now) || + p.Recipients != 3 || p.Sent != 0 || p.Failed != 0 || p.Dropped != 0 || p.Pending != 3 { + t.Fatalf("post: %+v", p) + } + for _, id := range outboxPostIDs(t, s) { + if id != p.ID { + t.Fatalf("every delivery carries the post id: %d != %d", id, p.ID) + } + } +} + +func TestEnqueuePostRecordsAPostRowWithNoRecipients(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + if err := s.EnqueuePost(l.ID, "mid:", []byte(postMsg), nil); err != nil { + t.Fatal(err) + } + if p := onlyPost(t, s, l.ID); p.Recipients != 0 || p.Pending != 0 { + t.Fatalf("post: %+v", p) + } +} + +func TestARefusedDuplicateRecordsNoPost(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.EnqueuePost(l.ID, "mid:", []byte(postMsg), items(1)) + if err := s.EnqueuePost(l.ID, "mid:", []byte(postMsg), items(1)); !errors.Is(err, ErrDuplicate) { + t.Fatalf("want ErrDuplicate, got %v", err) + } + onlyPost(t, s, l.ID) +} + +func TestAFailedQueueWriteRecordsNoPost(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + if _, err := s.db.Exec(`CREATE TRIGGER boom BEFORE INSERT ON outbox BEGIN SELECT RAISE(ABORT, 'boom'); END`); err != nil { + t.Fatal(err) + } + if err := s.EnqueuePost(l.ID, "mid:", []byte(postMsg), items(2)); err == nil { + t.Fatal("want the queue error") + } + hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r") + if err := s.EnqueueFromHeld(hid, "mid:", []byte(postMsg), items(2)); err == nil { + t.Fatal("want the queue error") + } + if posts, _ := s.ListPosts(l.ID, 0); len(posts) != 0 { + t.Fatalf("posts: %+v", posts) + } +} + +func TestReleasingAHeldPostRecordsItOnce(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + hid, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r") + if err := s.EnqueueFromHeld(hid, "mid:", []byte(postMsg), items(2)); err != nil { + t.Fatal(err) + } + s.EnqueueFromHeld(hid, "mid:", []byte(postMsg), items(2)) // a second approval: ErrNotFound + p := onlyPost(t, s, l.ID) + if p.Recipients != 2 || p.Subject != "Caffè news" { + t.Fatalf("post: %+v", p) + } + for _, id := range outboxPostIDs(t, s) { + if id != p.ID { + t.Fatalf("delivery post id %d != %d", id, p.ID) + } + } + hid2, _ := s.HoldMessage(l.ID, "a@x.com", []byte("x"), "r") + if err := s.EnqueueFromHeld(hid2, "mid:", []byte(postMsg), nil); err != nil { + t.Fatal(err) + } + if posts, _ := s.ListPosts(l.ID, 0); len(posts) != 2 || posts[0].Recipients != 0 { + t.Fatalf("a released post with no recipients has a row too: %+v", posts) + } +} + +func TestOtherMailIsNotAPost(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + if err := s.Enqueue([]byte(postMsg), items(2)); err != nil { + t.Fatal(err) + } + if posts, _ := s.ListPosts(l.ID, 0); len(posts) != 0 { + t.Fatalf("posts: %+v", posts) + } + for _, id := range outboxPostIDs(t, s) { + if id != 0 { + t.Fatalf("post id %d", id) + } + } +} + +func TestPostSubjectIsCutAndClean(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + msg := "Message-ID: \r\nSubject: " + strings.Repeat("é", 300) + "\r\n\r\nb" + s.EnqueuePost(l.ID, "mid:x", []byte(msg), items(1)) + p := onlyPost(t, s, l.ID) + if p.Subject != strings.Repeat("é", 200) || p.MessageID != "" { + t.Fatalf("post: %+v", p) + } +} + +func TestCountersFollowTheDeliveries(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + l, _ := s.CreateList("dev", "example.org", "Dev") + s.EnqueuePost(l.ID, "mid:", []byte(postMsg), items(6)) + due, _ := s.DueOutbox(10) + + if err := s.RetryOutbox(due[0].ID, now.Add(time.Minute), "down"); err != nil { + t.Fatal(err) + } + if p := onlyPost(t, s, l.ID); p.Sent+p.Failed+p.Dropped != 0 || p.Pending != 6 { + t.Fatalf("a retry changes nothing: %+v", p) + } + if err := s.CompleteOutbox(due[1].ID); err != nil { + t.Fatal(err) + } + if err := s.FailOutbox(due[2].ID, "refused"); err != nil { + t.Fatal(err) + } + if err := s.DropOutbox(due[3].ID); err != nil { + t.Fatal(err) + } + if err := s.DropOutbox(due[4].ID); err != nil { + t.Fatal(err) + } + p := onlyPost(t, s, l.ID) + if p.Sent != 1 || p.Failed != 1 || p.Dropped != 2 || p.Pending != 2 { + t.Fatalf("post: %+v", p) + } + // Discarding a failed delivery: it is already counted as failed. + if err := s.DropOutbox(due[2].ID); err != nil { + t.Fatal(err) + } + if err := s.CompleteOutbox(due[2].ID); !errors.Is(err, ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } + if err := s.DropOutbox(due[2].ID); !errors.Is(err, ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } + if p2 := onlyPost(t, s, l.ID); p2 != p { + t.Fatalf("discarding a failed delivery changes nothing: %+v", p2) + } +} + +func TestCompletingAFailedDeliveryDoesNotCountItAsSent(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.EnqueuePost(l.ID, "mid:", []byte(postMsg), items(1)) + due, _ := s.DueOutbox(10) + s.FailOutbox(due[0].ID, "refused") + if err := s.CompleteOutbox(due[0].ID); err != nil { + t.Fatal(err) + } + if p := onlyPost(t, s, l.ID); p.Sent != 0 || p.Failed != 1 || p.Pending != 0 { + t.Fatalf("post: %+v", p) + } +} + +func TestDropOutboxRemovesTheDeliveryAndTheBody(t *testing.T) { + s := newTestStore(t) + s.Enqueue([]byte("body"), items(1)) + due, _ := s.DueOutbox(10) + if err := s.DropOutbox(due[0].ID); err != nil { + t.Fatal(err) + } + if st, _ := s.OutboxStats(); st.Pending != 0 || st.Failed != 0 { + t.Fatalf("stats: %+v", st) + } + if _, err := s.OutboxData(due[0].MsgID); !errors.Is(err, ErrNotFound) { + t.Fatalf("body: %v", err) + } +} + +func TestPurgingOldFailuresKeepsTheCounters(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + l, _ := s.CreateList("dev", "example.org", "Dev") + s.EnqueuePost(l.ID, "mid:", []byte(postMsg), items(2)) + due, _ := s.DueOutbox(10) + s.FailOutbox(due[0].ID, "refused") + now = now.Add(8 * 24 * time.Hour) + if n, err := s.PurgeFailedOutbox(); err != nil || n != 1 { + t.Fatalf("purged %d %v", n, err) + } + if p := onlyPost(t, s, l.ID); p.Failed != 1 || p.Pending != 1 { + t.Fatalf("post: %+v", p) + } +} + +func TestListPostsNewestFirstWithALimit(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + dev, _ := s.CreateList("dev", "example.org", "Dev") + ops, _ := s.CreateList("ops", "example.org", "Ops") + for i := 0; i < 210; i++ { + now = now.Add(time.Second) + if err := s.EnqueuePost(dev.ID, "mid:"+string(rune(0x100+i)), []byte(postMsg), nil); err != nil { + t.Fatal(err) + } + } + s.EnqueuePost(ops.ID, "mid:ops", []byte(postMsg), items(1)) + all, _ := s.ListPosts(dev.ID, 1000) + if len(all) != 200 { + t.Fatalf("capped at 200: %d", len(all)) + } + for i := 1; i < len(all); i++ { + if !all[i-1].CreatedAt.After(all[i].CreatedAt) { + t.Fatalf("not newest first at %d", i) + } + } + if def, _ := s.ListPosts(dev.ID, 0); len(def) != 50 || def[0].ID != all[0].ID { + t.Fatalf("default 50: %d", len(def)) + } + if neg, _ := s.ListPosts(dev.ID, -1); len(neg) != 50 { + t.Fatalf("default 50: %d", len(neg)) + } + if some, _ := s.ListPosts(dev.ID, 7); len(some) != 7 { + t.Fatalf("limit 7: %d", len(some)) + } + if over, _ := s.ListPosts(dev.ID, 201); len(over) != 200 { + t.Fatalf("limit 201: %d", len(over)) + } + if exact, _ := s.ListPosts(dev.ID, 200); len(exact) != 200 { + t.Fatalf("limit 200: %d", len(exact)) + } + if o, _ := s.ListPosts(ops.ID, 0); len(o) != 1 || o[0].Recipients != 1 { + t.Fatalf("other list: %+v", o) + } +} + +func TestListPostsBreaksTiesByID(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.Now = func() time.Time { return time.Unix(1_000_000, 0) } + s.EnqueuePost(l.ID, "mid:1", []byte(postMsg), nil) + s.EnqueuePost(l.ID, "mid:2", []byte(postMsg), nil) + posts, _ := s.ListPosts(l.ID, 0) + if len(posts) != 2 || posts[0].ID < posts[1].ID { + t.Fatalf("the later post first: %+v", posts) + } +} + +func TestOldPostsArePurgedWhenANewOneIsRecorded(t *testing.T) { + s := newTestStore(t) + now := time.Unix(100_000_000, 0) + s.Now = func() time.Time { return now } + dev, _ := s.CreateList("dev", "example.org", "Dev") + ops, _ := s.CreateList("ops", "example.org", "Ops") + s.EnqueuePost(dev.ID, "mid:old", []byte(postMsg), nil) + now = now.Add(time.Second) + s.EnqueuePost(dev.ID, "mid:kept", []byte(postMsg), nil) + kept := now + now = now.Add(400 * 24 * time.Hour) // the first is now older than 400 days, the second exactly 400 days old + hid, _ := s.HoldMessage(ops.ID, "a@x.com", []byte("x"), "r") + if err := s.EnqueueFromHeld(hid, "mid:new", []byte(postMsg), nil); err != nil { + t.Fatal(err) + } + posts, _ := s.ListPosts(dev.ID, 0) + if len(posts) != 1 || !posts[0].CreatedAt.Equal(kept) { + t.Fatalf("posts: %+v", posts) + } + now = now.Add(400 * 24 * time.Hour) + s.EnqueuePost(ops.ID, "mid:newer", []byte(postMsg), nil) + if posts, _ := s.ListPosts(dev.ID, 0); len(posts) != 0 { + t.Fatalf("EnqueuePost purges too: %+v", posts) + } +} + +func TestDeletingAListDeletesItsPosts(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.EnqueuePost(l.ID, "mid:", []byte(postMsg), nil) + if err := s.DeleteList(l.ID); err != nil { + t.Fatal(err) + } + var n int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM posts`).Scan(&n); err != nil || n != 0 { + t.Fatalf("posts left: %d %v", n, err) + } +} + +func TestMigrationAddsPostAndPriorityColumns(t *testing.T) { + path := t.TempDir() + "/old.db" + s, _ := Open(path) + s.Enqueue([]byte("x"), items(1)) + for _, q := range []string{`DROP INDEX IF EXISTS outbox_due_prio`, `ALTER TABLE outbox DROP COLUMN post_id`, + `ALTER TABLE outbox DROP COLUMN priority`, `DROP TABLE posts`} { + if _, err := s.db.Exec(q); err != nil { + t.Fatalf("%s: %v", q, err) + } + } + s.Close() + s2, err := Open(path) + if err != nil { + t.Fatalf("migrate: %v", err) + } + defer s2.Close() + due, _ := s2.DueOutbox(10) + if len(due) != 1 { + t.Fatalf("an existing delivery stays waiting: %+v", due) + } + if ids := outboxPostIDs(t, s2); len(ids) != 1 || ids[0] != 0 { + t.Fatalf("post ids: %v", ids) + } + l, _ := s2.CreateList("dev", "example.org", "Dev") + if err := s2.EnqueuePost(l.ID, "mid:", []byte(postMsg), items(1)); err != nil { + t.Fatal(err) + } + onlyPost(t, s2, l.ID) +} + +func TestDueOutboxSendsLowerPriorityNumbersFirst(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + news := items(3) + for i := range news { + news[i].Priority = 1 + } + s.Enqueue([]byte("newsletter"), news) + now = now.Add(time.Minute) + s.Enqueue([]byte("discussion"), items(2)) + due, _ := s.DueOutbox(10) + if len(due) != 5 { + t.Fatalf("due: %d", len(due)) + } + for i, d := range due { + if d.Priority != map[bool]int{true: 0, false: 1}[i < 2] { + t.Fatalf("priority 0 first: %+v", due) + } + } + for i := 1; i < len(due); i++ { + if due[i].Priority == due[i-1].Priority && due[i].ID < due[i-1].ID { + t.Fatalf("FIFO within a priority: %+v", due) + } + } + if got, _ := s.DueOutbox(2); got[0].Priority != 0 || got[1].Priority != 0 { + t.Fatalf("the limit takes the first in priority: %+v", got) + } + // A priority-0 delivery that is not due yet does not jump ahead. + s.RetryOutbox(due[0].ID, now.Add(time.Hour), "down") + s.RetryOutbox(due[1].ID, now.Add(time.Hour), "down") + if got, _ := s.DueOutbox(10); len(got) != 3 || got[0].Priority != 1 { + t.Fatalf("due: %+v", got) + } +} + +func TestWithinAPriorityTheLongestWaitingGoesFirst(t *testing.T) { + s := newTestStore(t) + now := time.Unix(1_000_000, 0) + s.Now = func() time.Time { return now } + s.Enqueue([]byte("body"), items(3)) + due, _ := s.DueOutbox(10) + s.RetryOutbox(due[2].ID, now.Add(-time.Hour), "e") + got, _ := s.DueOutbox(10) + if got[0].ID != due[2].ID || got[1].ID != due[0].ID || got[2].ID != due[1].ID { + t.Fatalf("order: %+v", got) + } +} + +func TestFailingAFailedDeliveryAgainCountsItOnce(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.EnqueuePost(l.ID, "mid:", []byte(postMsg), items(1)) + due, _ := s.DueOutbox(10) + s.FailOutbox(due[0].ID, "refused") + if err := s.FailOutbox(due[0].ID, "refused again"); err != nil { + t.Fatal(err) + } + if p := onlyPost(t, s, l.ID); p.Failed != 1 || p.Pending != 0 { + t.Fatalf("post: %+v", p) + } +} diff --git a/internal/store/publisher_test.go b/internal/store/publisher_test.go new file mode 100644 index 0000000..93924b9 --- /dev/null +++ b/internal/store/publisher_test.go @@ -0,0 +1,126 @@ +package store + +import ( + "errors" + "testing" +) + +func TestANewMemberIsNotAPublisher(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("news", "example.org", "News") + s.AddMember(l.ID, "a@x.com", "") + ms, _ := s.Members(l.ID) + if len(ms) != 1 || ms[0].Publisher { + t.Fatalf("members: %+v", ms) + } + if ok, err := s.IsPublisher(l.ID, "a@x.com"); ok || err != nil { + t.Fatalf("IsPublisher: %v %v", ok, err) + } +} + +func TestSetPublisherPromotesAndDemotes(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("news", "example.org", "News") + s.AddMember(l.ID, "a@x.com", "") + s.AddMember(l.ID, "b@x.com", "") + if err := s.SetPublisher(l.ID, "A@X.com", true); err != nil { + t.Fatal(err) + } + if ok, err := s.IsPublisher(l.ID, "a@X.COM"); !ok || err != nil { + t.Fatalf("promoted: %v %v", ok, err) + } + if ok, _ := s.IsPublisher(l.ID, "b@x.com"); ok { + t.Fatal("only a is a publisher") + } + ms, _ := s.Members(l.ID) + if !ms[0].Publisher || ms[1].Publisher { + t.Fatalf("members: %+v", ms) + } + if m, _ := s.MemberByID(ms[0].ID); !m.Publisher { + t.Fatalf("by id: %+v", m) + } + if err := s.SetPublisher(l.ID, "a@x.com", false); err != nil { + t.Fatal(err) + } + if ok, _ := s.IsPublisher(l.ID, "a@x.com"); ok { + t.Fatal("demoted") + } +} + +func TestSetPublisherOfANonMemberIsNotFound(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("news", "example.org", "News") + other, _ := s.CreateList("dev", "example.org", "Dev") + s.AddMember(other.ID, "a@x.com", "") + if err := s.SetPublisher(l.ID, "a@x.com", true); !errors.Is(err, ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } + if ok, _ := s.IsPublisher(other.ID, "a@x.com"); ok { + t.Fatal("the flag is per list") + } +} + +func TestAPublisherDisabledByBouncesIsStillAPublisher(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("news", "example.org", "News") + s.AddMember(l.ID, "a@x.com", "") + s.SetPublisher(l.ID, "a@x.com", true) + ms, _ := s.Members(l.ID) + for i := 0; i < BounceThreshold; i++ { + s.RecordHardBounce(ms[0].ID, BounceThreshold) + } + if m, _ := s.MemberByID(ms[0].ID); !m.Disabled { + t.Fatal("setup: the member is disabled") + } + if ok, err := s.IsPublisher(l.ID, "a@x.com"); !ok || err != nil { + t.Fatalf("still a publisher: %v %v", ok, err) + } +} + +func TestReAddingAMemberKeepsThePublisherFlag(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("news", "example.org", "News") + s.AddMember(l.ID, "a@x.com", "") + s.SetPublisher(l.ID, "a@x.com", true) + if err := s.AddMember(l.ID, "a@x.com", "Alice"); err != nil { + t.Fatal(err) + } + ms, _ := s.Members(l.ID) + if len(ms) != 1 || !ms[0].Publisher || ms[0].DisplayName != "Alice" { + t.Fatalf("members: %+v", ms) + } +} + +func TestPublishersIsAPostPolicy(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("news", "example.org", "News") + if err := s.SetPolicies(l.ID, PostPublishers, JoinOpen, ArchivePublic); err != nil { + t.Fatal(err) + } + if got, _ := s.GetList(l.ID); got.PostPolicy != "publishers" { + t.Fatalf("policy: %+v", got) + } +} + +func TestMigrationAddsThePublisherColumn(t *testing.T) { + path := t.TempDir() + "/old.db" + s, _ := Open(path) + l, _ := s.CreateList("news", "example.org", "News") + s.AddMember(l.ID, "a@x.com", "A") + if _, err := s.db.Exec(`ALTER TABLE members DROP COLUMN publisher`); err != nil { + t.Fatal(err) + } + s.Close() + s2, err := Open(path) + if err != nil { + t.Fatalf("migrate: %v", err) + } + defer s2.Close() + ms, err := s2.Members(l.ID) + if err != nil || len(ms) != 1 || ms[0].Address != "a@x.com" || ms[0].Publisher { + t.Fatalf("an old member survives and is not a publisher: %+v %v", ms, err) + } + if err := s2.SetPublisher(l.ID, "a@x.com", true); err != nil { + t.Fatal(err) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 095b463..bfe150e 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -32,10 +32,11 @@ var ( // Policy values. const ( - PostMembers = "members" // only members may post, others are rejected - PostModerated = "moderated" // every post waits for approval - JoinOpen = "open" // anyone can subscribe (with confirmation) - JoinClosed = "closed" // only the admin adds members + PostMembers = "members" // only members may post, others are rejected + PostModerated = "moderated" // every post waits for approval + PostPublishers = "publishers" // newsletter: only members flagged as publishers may post + JoinOpen = "open" // anyone can subscribe (with confirmation) + JoinClosed = "closed" // only the admin adds members ) // Archive policy values. @@ -101,6 +102,7 @@ type Member struct { DisplayName string Disabled bool // too many hard bounces: no more delivery BounceCount int + Publisher bool // may post to a list whose post policy is PostPublishers } // Held is a message waiting for moderation. @@ -152,6 +154,7 @@ CREATE TABLE IF NOT EXISTS members ( disabled INTEGER NOT NULL DEFAULT 0, bounce_count INTEGER NOT NULL DEFAULT 0, bounce_at INTEGER NOT NULL DEFAULT 0, + publisher INTEGER NOT NULL DEFAULT 0, UNIQUE(list_id, address) ); CREATE TABLE IF NOT EXISTS held ( @@ -211,9 +214,24 @@ CREATE TABLE IF NOT EXISTS outbox ( next_at INTEGER NOT NULL, last_error TEXT NOT NULL DEFAULT '', created_at INTEGER NOT NULL, - failed INTEGER NOT NULL DEFAULT 0 + failed INTEGER NOT NULL DEFAULT 0, + post_id INTEGER NOT NULL DEFAULT 0, + priority INTEGER NOT NULL DEFAULT 0 ); CREATE INDEX IF NOT EXISTS outbox_due ON outbox(next_at, id); +CREATE INDEX IF NOT EXISTS outbox_due_prio ON outbox(priority, next_at, id); +CREATE TABLE IF NOT EXISTS posts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE, + message_id TEXT NOT NULL DEFAULT '', + subject TEXT NOT NULL DEFAULT '', + created_at INTEGER NOT NULL, + recipients INTEGER NOT NULL DEFAULT 0, + sent INTEGER NOT NULL DEFAULT 0, + failed INTEGER NOT NULL DEFAULT 0, + dropped INTEGER NOT NULL DEFAULT 0 +); +CREATE INDEX IF NOT EXISTS posts_list ON posts(list_id, created_at, id); CREATE TABLE IF NOT EXISTS delivered ( list_id INTEGER NOT NULL REFERENCES lists(id) ON DELETE CASCADE, post_id TEXT NOT NULL, @@ -292,6 +310,11 @@ func migrate(db *sql.DB) error { return err } } + if has, _ := hasColumn(db, "members", "publisher"); !has { + if _, err := db.Exec(`ALTER TABLE members ADD COLUMN publisher INTEGER NOT NULL DEFAULT 0`); err != nil { + return err + } + } } if ok, err := hasColumn(db, "outbox", "msg_id"); err != nil { return err @@ -301,6 +324,12 @@ func migrate(db *sql.DB) error { return err } } + if has, _ := hasColumn(db, "outbox", "post_id"); !has { + if _, err := db.Exec(`ALTER TABLE outbox ADD COLUMN post_id INTEGER NOT NULL DEFAULT 0; + ALTER TABLE outbox ADD COLUMN priority INTEGER NOT NULL DEFAULT 0;`); err != nil { + return err + } + } } if ok, err := hasColumn(db, "held", "list_id"); err != nil { return err @@ -389,23 +418,25 @@ func summarize(data []byte) (subject, from string) { if err != nil { return "", "" } - clean := func(s string) string { - s = strings.Map(func(r rune) rune { - if unicode.IsControl(r) { - return -1 - } - return r - }, strings.Join(strings.Fields(s), " ")) - if r := []rune(s); len(r) > 200 { - s = string(r[:200]) - } - return s - } raw := m.Header.Get("Subject") if dec, err := new(mime.WordDecoder).DecodeHeader(raw); err == nil { raw = dec } - return clean(raw), clean(m.Header.Get("From")) + return cleanSummary(raw), cleanSummary(m.Header.Get("From")) +} + +// cleanSummary makes a header value fit for a listing: one line without control characters, at most 200 characters. +func cleanSummary(s string) string { + s = strings.Map(func(r rune) rune { + if unicode.IsControl(r) { + return -1 + } + return r + }, strings.Join(strings.Fields(s), " ")) + if r := []rune(s); len(r) > 200 { + s = string(r[:200]) + } + return s } // Close closes the database. @@ -491,7 +522,7 @@ func (s *Store) UpdateList(id int64, title string) error { // SetPolicies changes the post, join and archive policies of a list. func (s *Store) SetPolicies(id int64, post, join, archive string) error { - if post != PostMembers && post != PostModerated { + if post != PostMembers && post != PostModerated && post != PostPublishers { return fmt.Errorf("invalid post policy %q", post) } if join != JoinOpen && join != JoinClosed { @@ -510,21 +541,22 @@ func (s *Store) DeleteList(id int64) error { // Members. -const memberCols = `id, list_id, address, display_name, disabled, bounce_count` +const memberCols = `id, list_id, address, display_name, disabled, bounce_count, publisher` func scanMember(row interface{ Scan(...any) error }) (Member, error) { var m Member - var dis int - err := row.Scan(&m.ID, &m.ListID, &m.Address, &m.DisplayName, &dis, &m.BounceCount) + var dis, pub int + err := row.Scan(&m.ID, &m.ListID, &m.Address, &m.DisplayName, &dis, &m.BounceCount, &pub) if errors.Is(err, sql.ErrNoRows) { return Member{}, ErrNotFound } m.Disabled = dis != 0 + m.Publisher = pub != 0 return m, err } -// AddMember subscribes an address to a list. Re-adding keeps the member id and -// clears the bounce state. +// AddMember subscribes an address to a list. Re-adding keeps the member id and the publisher +// flag and clears the bounce state. func (s *Store) AddMember(listID int64, addr, displayName string) error { return addMember(s.db, listID, addr, displayName) } @@ -543,6 +575,24 @@ func (s *Store) IsMember(listID int64, addr string) (bool, error) { return n > 0, err } +// SetPublisher flags or unflags a member as a publisher of the list. It returns ErrNotFound +// when the address is not a member. +func (s *Store) SetPublisher(listID int64, addr string, publisher bool) error { + v := 0 + if publisher { + v = 1 + } + return affected(s.db.Exec(`UPDATE members SET publisher=? WHERE list_id=? AND address=?`, v, listID, norm(addr))) +} + +// IsPublisher reports whether addr is a member flagged as a publisher of the list. A publisher +// disabled by bounces is still a publisher: bounces stop delivery to it, not its posts. +func (s *Store) IsPublisher(listID int64, addr string) (bool, error) { + var n int + err := s.db.QueryRow(`SELECT COUNT(*) FROM members WHERE list_id=? AND address=? AND publisher=1`, listID, norm(addr)).Scan(&n) + return n > 0, err +} + // IsActiveMember reports whether addr is subscribed and not disabled by bounces. func (s *Store) IsActiveMember(listID int64, addr string) (bool, error) { var n int diff --git a/internal/web/limit.go b/internal/web/limit.go new file mode 100644 index 0000000..d6b8e18 --- /dev/null +++ b/internal/web/limit.go @@ -0,0 +1,81 @@ +package web + +import ( + "net" + "net/http" + "sync" + "time" +) + +// A person subscribes once; a script posting the form for other people's addresses is what this +// slows down: at most subscribeMax POSTs per subscribeWindow from one address. The window starts +// at the first request of an address and everything about it is forgotten when it ends. +const ( + subscribeMax = 10 + subscribeWindow = time.Hour + sweepEvery = time.Minute +) + +type visits struct { + count int + first time.Time +} + +type subscribeLimiter struct { + mu sync.Mutex + now func() time.Time + byIP map[string]*visits + lastSweep time.Time +} + +func newSubscribeLimiter(now func() time.Time) *subscribeLimiter { + return &subscribeLimiter{now: now, byIP: map[string]*visits{}, lastSweep: now()} +} + +// allow counts a request from ip. When the address is over its limit it returns false and how long +// until it may try again; a refused request is not counted. +func (l *subscribeLimiter) allow(ip string) (retryAfter time.Duration, ok bool) { + l.mu.Lock() + defer l.mu.Unlock() + now := l.now() + l.sweep(now) + v := l.byIP[ip] + if v == nil || now.Sub(v.first) >= subscribeWindow { + v = &visits{first: now} + l.byIP[ip] = v + } + if v.count >= subscribeMax { + return v.first.Add(subscribeWindow).Sub(now), false + } + v.count++ + return 0, true +} + +// sweep forgets addresses whose window is over, so the map cannot grow forever. +func (l *subscribeLimiter) sweep(now time.Time) { + if now.Sub(l.lastSweep) < sweepEvery { + return + } + l.lastSweep = now + for ip, v := range l.byIP { + if now.Sub(v.first) >= subscribeWindow { + delete(l.byIP, ip) + } + } +} + +func (l *subscribeLimiter) size() int { + l.mu.Lock() + defer l.mu.Unlock() + return len(l.byIP) +} + +// clientIP is the address of the TCP peer. X-Forwarded-For is deliberately ignored: anyone can +// send it, so trusting it would let a script pick a fresh identity for every request. +func clientIP(r *http.Request) string { + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return r.RemoteAddr + } + return host +} diff --git a/internal/web/web.go b/internal/web/web.go index 1ed7d0f..b90cdf9 100644 --- a/internal/web/web.go +++ b/internal/web/web.go @@ -8,10 +8,12 @@ import ( "html/template" "log" "net/http" + "net/mail" "net/url" "strconv" "strings" "time" + "unicode" "codeberg.org/squidd/parsimony/assets" "codeberg.org/squidd/parsimony/internal/archive" @@ -28,11 +30,15 @@ type server struct { store *store.Store router *router.Router tmpl *template.Template + limit *subscribeLimiter } // New returns the public web handler. -func New(s *store.Store, rt *router.Router) http.Handler { - srv := &server{store: s, router: rt, tmpl: template.Must(template.New("").Funcs(template.FuncMap{ +func New(s *store.Store, rt *router.Router) http.Handler { return newHandler(s, rt, time.Now) } + +// newHandler is New with an injectable clock, for the tests. +func newHandler(s *store.Store, rt *router.Router, now func() time.Time) http.Handler { + srv := &server{store: s, router: rt, limit: newSubscribeLimiter(now), tmpl: template.Must(template.New("").Funcs(template.FuncMap{ "hide": archive.Obfuscate, // The logo is a trusted static asset: inline, so the strict CSP needs no image source for it. "mark": func() template.HTML { return template.HTML(assets.Mark) }, @@ -44,6 +50,8 @@ func New(s *store.Store, rt *router.Router) http.Handler { mux.HandleFunc("GET /{$}", srv.index) mux.HandleFunc("GET /archive/{address}", srv.archive) mux.HandleFunc("GET /archive/{address}/{id}", srv.message) + mux.HandleFunc("GET /subscribe/{address}", srv.subscribeForm) + mux.HandleFunc("POST /subscribe/{address}", srv.subscribe) mux.HandleFunc("GET /unsubscribe/{token}", srv.unsubscribePage) mux.HandleFunc("POST /unsubscribe/{token}", srv.unsubscribe) // Catch-all for GET so unknown paths get the HTML 404 page and other @@ -63,6 +71,9 @@ func secure(next http.Handler) http.Handler { if strings.HasPrefix(r.URL.Path, "/unsubscribe/") { h.Set("Cache-Control", "no-store") // the URL carries a secret } + if strings.HasPrefix(r.URL.Path, "/subscribe/") { + h.Set("Cache-Control", "no-store") // a form with an address typed into it + } next.ServeHTTP(w, r) }) } @@ -118,6 +129,8 @@ type archiveView struct { PrevURL string NextURL string ResultText string + // CanSubscribe is true when /subscribe/{address} serves this list. + CanSubscribe bool } func (s *server) archive(w http.ResponseWriter, r *http.Request) { @@ -152,7 +165,7 @@ func (s *server) archive(w http.ResponseWriter, r *http.Request) { return } - v := archiveView{List: l, Query: q, Messages: ms, Total: total, Page: page} + v := archiveView{List: l, Query: q, Messages: ms, Total: total, Page: page, CanSubscribe: l.JoinPolicy == store.JoinOpen} link := func(p int) string { vals := url.Values{} if q != "" { @@ -216,6 +229,108 @@ func (s *server) message(w http.ResponseWriter, r *http.Request) { s.render(w, http.StatusOK, "message", messageView{List: l, Message: m, Thread: thread}) } +// subscribableList resolves a list for the subscribe form: it must have a public archive and accept +// self-service subscriptions. Anything else is reported as missing, so a private or closed list +// cannot be told from one that does not exist. +func (s *server) subscribableList(address string) (store.List, bool, error) { + l, ok, err := s.publicList(address) + if err != nil || !ok || l.JoinPolicy != store.JoinOpen { + return store.List{}, false, err + } + return l, true, nil +} + +type subscribeView struct { + List store.List + Email string + Error string +} + +func (s *server) subscribeForm(w http.ResponseWriter, r *http.Request) { + l, ok, err := s.subscribableList(r.PathValue("address")) + switch { + case err != nil: + s.fail(w, err) + case !ok: + s.notFound(w) + default: + s.render(w, http.StatusOK, "subscribe", subscribeView{List: l}) + } +} + +// maxSubscribeBody is plenty for an email address and a honeypot field. +const maxSubscribeBody = 4 << 10 + +func (s *server) subscribe(w http.ResponseWriter, r *http.Request) { + if wait, ok := s.limit.allow(clientIP(r)); !ok { + w.Header().Set("Retry-After", strconv.Itoa(int((wait+time.Second-1)/time.Second))) + s.render(w, http.StatusTooManyRequests, "toomany", nil) + return + } + l, ok, err := s.subscribableList(r.PathValue("address")) + if err != nil { + s.fail(w, err) + return + } + if !ok { + s.notFound(w) + return + } + r.Body = http.MaxBytesReader(w, r.Body, maxSubscribeBody) + if err := r.ParseForm(); err != nil { + var tooBig *http.MaxBytesError + if errors.As(err, &tooBig) { + http.Error(w, "request too large", http.StatusRequestEntityTooLarge) + return + } + } + // A bot that fills every field gets the page a person gets, and nothing happens. + if r.PostForm.Get("website") != "" { + s.render(w, http.StatusOK, "subscribed", nil) + return + } + typed := r.PostForm.Get("email") + addr, ok := plainAddress(typed) + if !ok { + s.render(w, http.StatusBadRequest, "subscribe", subscribeView{List: l, Email: printable(typed), Error: "Enter a valid email address."}) + return + } + if err := s.router.RequestJoin(l.Address(), addr); err != nil { + s.fail(w, err) + return + } + s.render(w, http.StatusOK, "subscribed", nil) +} + +// printable drops control characters, so what was typed can be shown back safely. +func printable(s string) string { + return strings.Map(func(r rune) rune { + if unicode.IsControl(r) { + return -1 + } + return r + }, s) +} + +// plainAddress accepts one bare ASCII address of at most 254 characters: no display name, no angle +// brackets, no list, no quoting, no control characters. +func plainAddress(s string) (string, bool) { + s = strings.TrimSpace(s) + if s == "" || len(s) > 254 { + return "", false + } + for _, c := range []byte(s) { + if c <= ' ' || c >= 0x7f { + return "", false + } + } + a, err := mail.ParseAddress(s) + if err != nil || a.Name != "" || a.Address != s { + return "", false + } + return s, true +} + func (s *server) unsubscribePage(w http.ResponseWriter, r *http.Request) { l, err := s.router.CheckUnsubscribe(r.PathValue("token")) switch { @@ -259,6 +374,9 @@ pre{white-space:pre-wrap;word-wrap:break-word;background:#f6f6f6;padding:1rem;bo input[type=search]{padding:.35rem;width:16rem} button{padding:.5rem 1rem;font-size:1rem} nav a{margin-right:1rem} +.error{color:#b00020} +.hp{position:absolute;left:-10000px;width:1px;height:1px;overflow:hidden} +input[type=email]{padding:.35rem;width:20rem;max-width:100%} .brand{display:inline-block;margin-bottom:1rem} .brand svg{width:2rem;height:2rem;display:block} @@ -272,7 +390,7 @@ nav a{margin-right:1rem} {{template "foot"}}{{end}} {{define "archive"}}{{template "head" .List.Title}} - +

{{.List.Title}}

{{.List.Address}}

@@ -294,6 +412,28 @@ nav a{margin-right:1rem} {{hide .Sender}}, {{date .Date}}{{end}}{{end}} {{template "foot"}}{{end}} +{{define "subscribe"}}{{template "head" (print "Subscribe to " .List.Title)}} + +

Subscribe to {{.List.Title}}

+

{{.List.Address}}

+{{if .Error}}{{end}} +
+

+ + +
+

We send one message to confirm the address. Nothing happens until you reply to it.

+{{template "foot"}}{{end}} + +{{define "subscribed"}}{{template "head" "Check your email"}} +

Check your email

+

If this address can subscribe, a confirmation message is on its way. Open it and reply to confirm.

+{{template "foot"}}{{end}} + +{{define "toomany"}}{{template "head" "Too many requests"}} +

Too many requests

Please try again later.

+{{template "foot"}}{{end}} + {{define "unsubscribe"}}{{template "head" "Unsubscribe"}}

Unsubscribe

Stop receiving messages from {{.Title}} ({{.Address}})?

diff --git a/internal/web/web_test.go b/internal/web/web_test.go index 2e517f5..62fae73 100644 --- a/internal/web/web_test.go +++ b/internal/web/web_test.go @@ -1,17 +1,21 @@ package web import ( + "context" "encoding/base64" "fmt" "html" "net/http" "net/http/httptest" "net/url" + "strconv" "strings" "testing" "time" "codeberg.org/squidd/parsimony/assets" + "codeberg.org/squidd/parsimony/internal/outbound" + "codeberg.org/squidd/parsimony/internal/queue" "codeberg.org/squidd/parsimony/internal/router" "codeberg.org/squidd/parsimony/internal/store" ) @@ -23,6 +27,8 @@ type fixture struct { pub store.List alice int64 bobTok string + out *outbound.Fake + clock time.Time // what the handler thinks the time is; tests move it } func newFixture(t *testing.T) *fixture { @@ -38,12 +44,32 @@ func newFixture(t *testing.T) *fixture { s.SetPolicies(priv.ID, store.PostMembers, store.JoinOpen, store.ArchivePrivate) off, _ := s.CreateList("off", "example.org", "Off") s.SetPolicies(off.ID, store.PostMembers, store.JoinOpen, store.ArchiveOff) + closed, _ := s.CreateList("closed", "example.org", "Closed") + s.SetPolicies(closed.ID, store.PostMembers, store.JoinClosed, store.ArchivePublic) s.AddMember(pub.ID, "alice@example.com", "Alice") s.AddMember(pub.ID, "bob@example.com", "Bob") ms, _ := s.Members(pub.ID) rt := router.New(s) - return &fixture{h: New(s, rt), s: s, rt: rt, pub: pub, alice: ms[0].ID, bobTok: rt.UnsubscribeToken(ms[1].ID)} + f := &fixture{s: s, rt: rt, pub: pub, alice: ms[0].ID, bobTok: rt.UnsubscribeToken(ms[1].ID), + out: &outbound.Fake{}, clock: time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)} + f.h = newHandler(s, rt, func() time.Time { return f.clock }) + return f +} + +// flush sends what is queued, as the background worker would, and returns everything sent so far. +func (f *fixture) flush(t *testing.T) []outbound.Sent { + t.Helper() + w := &queue.Worker{Store: f.s, Sender: f.out} + for { + st, err := w.RunOnce(context.Background()) + if err != nil { + t.Fatal(err) + } + if st == (queue.Stats{}) { + return f.out.Sent + } + } } func (f *fixture) add(t *testing.T, listID int64, id, subject, sender, body string, at int64, inReplyTo string) int64 { @@ -324,3 +350,367 @@ func TestTheCSPAllowsOnlyDataImages(t *testing.T) { } } } + +// Subscribe form. + +const neutralText = "If this address can subscribe, a confirmation message is on its way. Open it and reply to confirm." + +// subscribe posts a form from the TCP peer ip. +func subscribe(t *testing.T, f *fixture, list, ip string, form url.Values) *httptest.ResponseRecorder { + t.Helper() + return subscribeRaw(f.h, list, ip, form.Encode(), nil) +} + +func subscribeRaw(h http.Handler, list, ip, body string, hdr map[string]string) *httptest.ResponseRecorder { + req := httptest.NewRequest("POST", "/subscribe/"+list, strings.NewReader(body)) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + for k, v := range hdr { + req.Header.Set(k, v) + } + req.RemoteAddr = ip + ":4321" + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + return rec +} + +func email(addr string) url.Values { return url.Values{"email": {addr}} } + +func TestSubscribeFormIsShownForPublicOpenLists(t *testing.T) { + f := newFixture(t) + rec := get(t, f.h, "/subscribe/dev@example.org") + wantStatus(t, rec, 200) + wantBody(t, rec, `method="post"`, `action="/subscribe/dev@example.org"`, `name="email"`, `type="email"`, + `name="website"`, "Dev <b>talk</b>", "") + wantNoBody(t, rec, "talk") + if got := rec.Header().Get("Cache-Control"); got != "no-store" { + t.Errorf("Cache-Control = %q, want no-store", got) + } + if csp := rec.Header().Get("Content-Security-Policy"); csp != "default-src 'none'; img-src data:; style-src 'unsafe-inline'; form-action 'self'; base-uri 'none'; frame-ancestors 'none'" { + t.Errorf("the CSP must not change: %q", csp) + } + // The honeypot is hidden from people and from assistive technology, and nothing fills it in. + wantBody(t, rec, `class="hp"`, `tabindex="-1"`, `autocomplete="off"`, `aria-hidden="true"`) +} + +func TestSubscribeIs404ForEverythingElseAndLooksLikeAMissingList(t *testing.T) { + f := newFixture(t) + missing := get(t, f.h, "/subscribe/ghost@example.org") + wantStatus(t, missing, 404) + for _, p := range []string{ + "/subscribe/secret@example.org", // private archive + "/subscribe/off@example.org", // no archive + "/subscribe/closed@example.org", // public archive, nobody may join + "/subscribe/not-an-address", + "/subscribe/..%2Fdev@example.org", + "/subscribe/dev@example.org%2F..%2Fsecret@example.org", + "/subscribe/dev@example.org%00", + "/subscribe/dev@example.org/extra", + "/subscribe/", + "/subscribe", + } { + rec := get(t, f.h, p) + wantStatus(t, rec, 404) + if rec.Body.String() != missing.Body.String() { + t.Errorf("%s: the 404 page must be the same as for a missing list", p) + } + rec = request(t, f.h, "POST", p) + if rec.Code != 404 && rec.Code != 405 { + t.Errorf("POST %s = %d, want 404 or 405", p, rec.Code) + } + } +} + +func TestSubscribeQueuesOneConfirmationMail(t *testing.T) { + f := newFixture(t) + rec := subscribe(t, f, "dev@example.org", "192.0.2.1", email("carol@x.com")) + wantStatus(t, rec, 200) + wantBody(t, rec, neutralText) + wantNoBody(t, rec, "carol") + if got := rec.Header().Get("Cache-Control"); got != "no-store" { + t.Errorf("Cache-Control = %q, want no-store", got) + } + sent := f.flush(t) + if len(sent) != 1 || len(sent[0].To) != 1 || sent[0].To[0] != "carol@x.com" { + t.Fatalf("want one confirmation mail to carol, got %+v", sent) + } + if !strings.Contains(string(sent[0].Data), "Reply-To: dev+confirm-") { + t.Fatalf("not a confirmation mail: %s", sent[0].Data) + } + if ok, _ := f.s.IsMember(f.pub.ID, "carol@x.com"); ok { + t.Fatal("not a member before confirming") + } + // Exactly one pending confirmation: the store refuses a second one within the hour. + subscribe(t, f, "dev@example.org", "192.0.2.1", email("carol@x.com")) + if sent := f.flush(t); len(sent) != 1 { + t.Fatalf("a repeat must not queue another mail, sent %d", len(sent)) + } +} + +func TestSubscribeAnswersTheSameWhateverHappens(t *testing.T) { + f := newFixture(t) + f.s.AddMember(f.pub.ID, "member@x.com", "") + fresh := subscribe(t, f, "dev@example.org", "192.0.2.1", email("new@x.com")) + member := subscribe(t, f, "dev@example.org", "192.0.2.2", email("member@x.com")) + limited := subscribe(t, f, "dev@example.org", "192.0.2.3", email("new@x.com")) // the store rate limits it + if fresh.Code != 200 || member.Code != 200 || limited.Code != 200 { + t.Fatalf("statuses %d %d %d", fresh.Code, member.Code, limited.Code) + } + for name, rec := range map[string]*httptest.ResponseRecorder{"member": member, "rate limited": limited} { + if rec.Body.String() != fresh.Body.String() { + t.Errorf("%s: the page differs from the page for a new address", name) + } + } + wantBody(t, fresh, neutralText) + if sent := f.flush(t); len(sent) != 1 || sent[0].To[0] != "new@x.com" { + t.Fatalf("only the new address gets mail: %+v", sent) + } +} + +func TestSubscribeAcceptsTheAddressInAnyCase(t *testing.T) { + f := newFixture(t) + f.s.AddMember(f.pub.ID, "member@x.com", "") + subscribe(t, f, "dev@example.org", "192.0.2.1", email(" MEMBER@X.com ")) + if sent := f.flush(t); len(sent) != 0 { + t.Fatalf("a member gets no mail whatever the case: %+v", sent) + } +} + +func TestSubscribeHoneypotDoesNothing(t *testing.T) { + f := newFixture(t) + good := subscribe(t, f, "dev@example.org", "192.0.2.1", email("ok@x.com")) + f.flush(t) + f.out.Sent = nil + bot := subscribe(t, f, "dev@example.org", "192.0.2.2", url.Values{"email": {"bot@x.com"}, "website": {"http://spam"}}) + wantStatus(t, bot, 200) + if bot.Body.String() != good.Body.String() { + t.Error("a bot must see the page a person sees") + } + // Even a bad address gets the success page when the honeypot is filled. + bot = subscribe(t, f, "dev@example.org", "192.0.2.2", url.Values{"email": {"nonsense"}, "website": {"x"}}) + wantStatus(t, bot, 200) + wantBody(t, bot, neutralText) + if sent := f.flush(t); len(sent) != 0 { + t.Fatalf("the honeypot must queue nothing: %+v", sent) + } + // No pending confirmation was created either: a real request for the same address still works. + subscribe(t, f, "dev@example.org", "192.0.2.3", email("bot@x.com")) + if sent := f.flush(t); len(sent) != 1 { + t.Fatalf("a real request after the honeypot must send its mail, sent %d", len(sent)) + } +} + +func TestSubscribeRejectsBadAddressesAndEscapesWhatWasTyped(t *testing.T) { + f := newFixture(t) + long := strings.Repeat("a", 250) + "@x.com" // 256 characters + for i, c := range []struct{ name, in string }{ + {"empty", ""}, + {"blank", " "}, + {"no at sign", "carol"}, + {"display name", "Carol "}, + {"angle brackets only", ""}, + {"two addresses", "a@x.com, b@x.com"}, + {"two addresses with a space", "a@x.com b@x.com"}, + {"markup", `">`}, + {"markup in an address", `@x.com`}, + {"quoted local part", `"a b"@x.com`}, + {"control character", "a\x01@x.com"}, + {"line break", "a@x.com\r\nBcc: v@x.com"}, + {"non-ASCII", "cärol@x.com"}, + {"too long", long}, + } { + rec := subscribe(t, f, "dev@example.org", fmt.Sprintf("192.0.2.%d", i+1), email(c.in)) // one IP each: the limit is per IP + if rec.Code != 400 { + t.Errorf("%s: status = %d, want 400", c.name, rec.Code) + } + wantBody(t, rec, `name="email"`, "valid email address") + wantNoBody(t, rec, neutralText, "`)) + wantBody(t, rec, `value="<script>alert(1)</script>"`) + // A 254-character address is the longest accepted. + ok := strings.Repeat("a", 64) + "@" + strings.Repeat("b", 63) + "." + strings.Repeat("c", 63) + "." + strings.Repeat("d", 57) + ".com" + if len(ok) != 254 { + t.Fatalf("setup: %d", len(ok)) + } + wantBody(t, subscribe(t, f, "dev@example.org", "192.0.2.101", email(ok)), neutralText) + wantBody(t, subscribe(t, f, "dev@example.org", "192.0.2.101", email("a"+ok)), "valid email address") +} + +func TestSubscribeBodyIsLimitedTo4KiB(t *testing.T) { + f := newFixture(t) + body := func(n int) string { + head := "email=carol%40x.com&pad=" + return head + strings.Repeat("x", n-len(head)) + } + rec := subscribeRaw(f.h, "dev@example.org", "192.0.2.1", body(4096), nil) + wantStatus(t, rec, 200) + wantBody(t, rec, neutralText) + rec = subscribeRaw(f.h, "dev@example.org", "192.0.2.1", body(4097), nil) + wantStatus(t, rec, http.StatusRequestEntityTooLarge) + wantNoBody(t, rec, neutralText) + rec = subscribeRaw(f.h, "dev@example.org", "192.0.2.1", body(1<<20), nil) + wantStatus(t, rec, http.StatusRequestEntityTooLarge) + if sent := f.flush(t); len(sent) != 1 { + t.Fatalf("only the small request sends mail, sent %d", len(sent)) + } +} + +func TestSubscribeNeverMailsTheAddressForAClosedOrHiddenList(t *testing.T) { + f := newFixture(t) + for _, l := range []string{"closed@example.org", "secret@example.org", "off@example.org", "ghost@example.org"} { + rec := subscribe(t, f, l, "192.0.2.1", email("victim@x.com")) + wantStatus(t, rec, 404) + wantNoBody(t, rec, neutralText) + } + if sent := f.flush(t); len(sent) != 0 { + t.Fatalf("no mail for these lists: %+v", sent) + } + // Even after the list opens, nothing was left pending. + cl, _ := f.s.FindList("closed@example.org") + f.s.SetPolicies(cl.ID, store.PostMembers, store.JoinOpen, store.ArchivePublic) + subscribe(t, f, "closed@example.org", "192.0.2.1", email("victim@x.com")) + if sent := f.flush(t); len(sent) != 1 { + t.Fatalf("one mail once open, sent %d", len(sent)) + } +} + +func TestSubscribeIsLimitedPerIPAndTheLimitExpires(t *testing.T) { + f := newFixture(t) + for i := 0; i < 10; i++ { + wantStatus(t, subscribe(t, f, "dev@example.org", "198.51.100.7", email(fmt.Sprintf("u%d@x.com", i))), 200) + } + f.clock = f.clock.Add(10 * time.Minute) + rec := subscribe(t, f, "dev@example.org", "198.51.100.7", email("u10@x.com")) + wantStatus(t, rec, 429) + if got := rec.Header().Get("Retry-After"); got != strconv.Itoa(50*60) { + t.Errorf("Retry-After = %q, want %d", got, 50*60) + } + wantNoBody(t, rec, neutralText) + // A refused request is not counted again, and queued nothing. + if sent := f.flush(t); len(sent) != 10 { + t.Fatalf("sent %d, want 10", len(sent)) + } + // Another address is not affected. + wantStatus(t, subscribe(t, f, "dev@example.org", "198.51.100.8", email("other@x.com")), 200) + // The window is an hour from the first request. + f.clock = f.clock.Add(49*time.Minute + 59*time.Second) + wantStatus(t, subscribe(t, f, "dev@example.org", "198.51.100.7", email("u11@x.com")), 429) + f.clock = f.clock.Add(2 * time.Second) + wantStatus(t, subscribe(t, f, "dev@example.org", "198.51.100.7", email("u11@x.com")), 200) +} + +func TestSubscribeLimitCountsEveryPostOfTheAddress(t *testing.T) { + f := newFixture(t) + // Honeypot hits, bad input and unknown lists all count, so they cannot be used to probe for free. + forms := []url.Values{{"email": {"x"}}, {"email": {"a@x.com"}, "website": {"x"}}, email("b@x.com")} + for i := 0; i < 10; i++ { + subscribe(t, f, []string{"dev@example.org", "ghost@example.org"}[i%2], "198.51.100.9", forms[i%3]) + } + wantStatus(t, subscribe(t, f, "dev@example.org", "198.51.100.9", email("c@x.com")), 429) +} + +func TestSubscribeLimitIgnoresForwardedFor(t *testing.T) { + f := newFixture(t) + for i := 0; i < 10; i++ { + hdr := map[string]string{"X-Forwarded-For": fmt.Sprintf("203.0.113.%d", i), "X-Real-IP": fmt.Sprintf("203.0.113.%d", i)} + wantStatus(t, subscribeRaw(f.h, "dev@example.org", "198.51.100.7", email("a@x.com").Encode(), hdr), 200) + } + hdr := map[string]string{"X-Forwarded-For": "203.0.113.200", "X-Real-IP": "203.0.113.200", "Forwarded": "for=203.0.113.201"} + wantStatus(t, subscribeRaw(f.h, "dev@example.org", "198.51.100.7", email("a@x.com").Encode(), hdr), 429) + // And naming the blocked address in the header does not hurt somebody else. + hdr = map[string]string{"X-Forwarded-For": "198.51.100.7"} + wantStatus(t, subscribeRaw(f.h, "dev@example.org", "198.51.100.99", email("a@x.com").Encode(), hdr), 200) +} + +func TestSubscribeLimiterForgetsIdleAddresses(t *testing.T) { + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + l := newSubscribeLimiter(func() time.Time { return now }) + for i := 0; i < 50; i++ { + l.allow(fmt.Sprintf("192.0.2.%d", i)) + } + if l.size() != 50 { + t.Fatalf("size = %d, want 50", l.size()) + } + now = now.Add(30 * time.Minute) + l.allow("198.51.100.1") + if l.size() != 51 { + t.Fatalf("addresses inside the window are kept, size = %d", l.size()) + } + now = now.Add(31 * time.Minute) // the first 50 are now older than an hour + l.allow("198.51.100.2") + if l.size() != 2 { + t.Fatalf("idle addresses must be forgotten, size = %d, want 2", l.size()) + } +} + +func TestSubscribeFailsWhenTheDatabaseDoes(t *testing.T) { + f := newFixture(t) + f.s.Close() + wantStatus(t, subscribe(t, f, "dev@example.org", "192.0.2.1", email("a@x.com")), 500) + wantStatus(t, get(t, f.h, "/subscribe/dev@example.org"), 500) +} + +func TestSubscribeLinkOnlyOnTheArchivePageOfAQualifyingList(t *testing.T) { + f := newFixture(t) + link := `href="/subscribe/dev@example.org"` + rec := get(t, f.h, "/archive/dev@example.org") + wantBody(t, rec, link, ">Subscribe") + // A public archive whose list is closed offers none. + wantNoBody(t, get(t, f.h, "/archive/closed@example.org"), "/subscribe/") + // Nowhere else. + id := f.add(t, f.pub.ID, "<1@x>", "Topic", "Alice", "text", 1_700_000_000, "") + wantNoBody(t, get(t, f.h, "/"), "/subscribe/") + wantNoBody(t, get(t, f.h, fmt.Sprintf("/archive/dev@example.org/%d", id)), "/subscribe/") + wantNoBody(t, get(t, f.h, "/unsubscribe/"+f.bobTok), "/subscribe/") +} + +func TestSubscribeTrimsTheTypedAddress(t *testing.T) { + f := newFixture(t) + f.s.AddMember(f.pub.ID, "member@x.com", "") + wantBody(t, subscribe(t, f, "dev@example.org", "192.0.2.1", email(" carol@x.com\t")), neutralText) + wantBody(t, subscribe(t, f, "dev@example.org", "192.0.2.2", email(" MEMBER@x.com ")), neutralText) + sent := f.flush(t) + if len(sent) != 1 || sent[0].To[0] != "carol@x.com" { + t.Fatalf("one mail, to the trimmed address: %+v", sent) + } +} + +func TestSubscribeRetryAfterRoundsUp(t *testing.T) { + f := newFixture(t) + for i := 0; i < 10; i++ { + subscribe(t, f, "dev@example.org", "198.51.100.7", email("a@x.com")) + } + f.clock = f.clock.Add(10*time.Minute + 500*time.Millisecond) + rec := subscribe(t, f, "dev@example.org", "198.51.100.7", email("a@x.com")) + wantStatus(t, rec, 429) + if got := rec.Header().Get("Retry-After"); got != "3000" { // 2999.5 s left + t.Errorf("Retry-After = %q, want 3000", got) + } +} + +// The hour is over when an hour has passed, not one tick later. +func TestSubscribeWindowEndsAfterExactlyAnHour(t *testing.T) { + f := newFixture(t) + for i := 0; i < 10; i++ { + subscribe(t, f, "dev@example.org", "198.51.100.7", email("a@x.com")) + } + f.clock = f.clock.Add(time.Hour - time.Nanosecond) + wantStatus(t, subscribe(t, f, "dev@example.org", "198.51.100.7", email("a@x.com")), 429) + f.clock = f.clock.Add(time.Nanosecond) + wantStatus(t, subscribe(t, f, "dev@example.org", "198.51.100.7", email("a@x.com")), 200) +} + +func TestSubscribeLimiterSweepsAnAddressWhoseHourIsOver(t *testing.T) { + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + l := newSubscribeLimiter(func() time.Time { return now }) + l.allow("192.0.2.1") + now = now.Add(time.Hour) // exactly an hour since the first request + l.allow("192.0.2.2") + if l.size() != 1 { + t.Fatalf("size = %d, want 1 (the first address is forgotten)", l.size()) + } +} diff --git a/parsimony.example.yaml b/parsimony.example.yaml index aff42fd..216e426 100644 --- a/parsimony.example.yaml +++ b/parsimony.example.yaml @@ -12,6 +12,7 @@ relay: # username: user # password_file: /run/secrets/relay-password concurrency: 4 # simultaneous deliveries + # rate_per_second: 14 # at most this many sends started per second, for relays that throttle (Amazon SES); 0 = unlimited web: listen: 0.0.0.0:8081 # public archive and unsubscribe pages; "" disables them api: