Merge pull request 'Import archives from mbox files (Mailman migration)' (#23) from feat/archive-import into main
This commit is contained in:
commit
fb313736bb
19 changed files with 1684 additions and 20 deletions
18
README.md
18
README.md
|
|
@ -61,6 +61,7 @@ parsimony member import dev@example.org subscribers.txt # "email[,name]" per l
|
|||
parsimony held ls dev@example.org
|
||||
parsimony held approve 12
|
||||
parsimony archive ls dev@example.org --q "deploy"
|
||||
parsimony archive import dev@example.org dev.mbox # an mbox file or "-" for stdin
|
||||
parsimony queue ls
|
||||
parsimony --json list ls # JSON for scripts
|
||||
parsimony help
|
||||
|
|
@ -71,6 +72,23 @@ Exit status: 0 success, 1 failure (including lines that failed in `member import
|
|||
`parsimony serve -config FILE` runs the daemon; the old `parsimony -config FILE` form still works.
|
||||
In the container: `docker exec parsimony /parsimony list ls`.
|
||||
|
||||
## Migrating from Mailman
|
||||
|
||||
Both Mailman 2 and Mailman 3 (HyperKitty) can export a list's history as an mbox file.
|
||||
|
||||
```
|
||||
parsimony list add dev@example.org --title "Developers"
|
||||
parsimony member import dev@example.org subscribers.txt # "email[,name]" per line
|
||||
parsimony archive import dev@example.org dev.mbox
|
||||
```
|
||||
|
||||
The import reads standard mbox (a message starts at a `From addr Tue Jan 2 15:04:05 2024` line, `>From ` escapes are
|
||||
restored), decodes encoded headers, charsets and multipart bodies, and stores the plain text. It is safe to repeat: a
|
||||
message already in the archive (same `Message-ID`) is skipped. Threads are rebuilt from `In-Reply-To` whatever the order
|
||||
of the messages. Messages that cannot be read are listed and counted, and the exit status is 1; the others are kept.
|
||||
It works whatever the list's archive policy is (the policy only governs live mail). The admin UI has the same import on
|
||||
the Archive tab. Search is accent-insensitive: `caffe` finds "Caffè".
|
||||
|
||||
## Docker
|
||||
|
||||
The image runs as an unprivileged user (uid 65532) and works with a read-only root filesystem.
|
||||
|
|
|
|||
|
|
@ -159,4 +159,17 @@ describe('endpoints', () => {
|
|||
['GET', '/api/v1/queue'],
|
||||
])
|
||||
})
|
||||
|
||||
it('uploads an mbox as the raw body with its own content type', async () => {
|
||||
mockFetch(respond(200, { imported: 2, duplicates: 0, failed: 0, errors: [] }))
|
||||
const file = new File(['From a@x.com Tue Jan 2 15:04:05 2024\nSubject: x\n\nbody\n'], 'dev.mbox')
|
||||
const sum = await api.archive.import(7, file)
|
||||
expect(sum).toEqual({ imported: 2, duplicates: 0, failed: 0, errors: [] })
|
||||
expect(calls[0].url).toBe('/api/v1/lists/7/archive/import')
|
||||
expect(calls[0].init.method).toBe('POST')
|
||||
expect(calls[0].init.body).toBe(file) // streamed as is, not JSON-encoded
|
||||
const headers = calls[0].init.headers as Record<string, string>
|
||||
expect(headers['Content-Type']).toBe('application/mbox')
|
||||
expect(headers.Authorization).toBe('Bearer secret-token')
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -42,6 +42,13 @@ export interface Message {
|
|||
body?: string
|
||||
}
|
||||
|
||||
export interface ImportSummary {
|
||||
imported: number
|
||||
duplicates: number
|
||||
failed: number
|
||||
errors: string[]
|
||||
}
|
||||
|
||||
export interface ArchivePage {
|
||||
total: number
|
||||
messages: Message[]
|
||||
|
|
@ -90,14 +97,16 @@ async function request<T>(method: string, path: string, body?: unknown): Promise
|
|||
const headers: Record<string, string> = {}
|
||||
const token = session.get()
|
||||
if (token) headers.Authorization = `Bearer ${token}`
|
||||
if (body !== undefined) headers['Content-Type'] = 'application/json'
|
||||
// A file is streamed as is; everything else is JSON.
|
||||
const raw = body instanceof Blob
|
||||
if (body !== undefined) headers['Content-Type'] = raw ? 'application/mbox' : 'application/json'
|
||||
|
||||
let res: Response
|
||||
try {
|
||||
res = await fetch(`/api/v1${path}`, {
|
||||
method,
|
||||
headers,
|
||||
body: body === undefined ? undefined : JSON.stringify(body),
|
||||
body: body === undefined ? undefined : raw ? body : JSON.stringify(body),
|
||||
})
|
||||
} catch {
|
||||
throw new ApiError(0, 'Cannot reach the server')
|
||||
|
|
@ -159,6 +168,7 @@ export const api = {
|
|||
const s = qs.toString()
|
||||
return request<ArchivePage>('GET', `/lists/${listId}/archive${s ? `?${s}` : ''}`)
|
||||
},
|
||||
import: (listId: number, file: Blob) => request<ImportSummary>('POST', `/lists/${listId}/archive/import`, file),
|
||||
get: (id: number) => request<Message>('GET', `/archive/${id}`),
|
||||
remove: (id: number) => request<void>('DELETE', `/archive/${id}`),
|
||||
},
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { screen, waitFor, within } from '@testing-library/react'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { Message } from '@/lib/api'
|
||||
import { aList, installFakeApi, newState } from '@/test/fakeApi'
|
||||
import { renderApp } from '@/test/render'
|
||||
|
|
@ -171,4 +171,114 @@ describe('archive tab', () => {
|
|||
expect(state.archive[1]).toHaveLength(2)
|
||||
})
|
||||
})
|
||||
|
||||
describe('importing an mbox', () => {
|
||||
const entry = (n: number) =>
|
||||
`From user${n}@x.com Tue Jan 2 15:04:05 2024\nFrom: User ${n} <user${n}@x.com>\nSubject: Imported ${n}\nMessage-ID: <imp${n}@x>\n\nbody ${n}\n\n`
|
||||
const mboxFile = (...ns: number[]) => new File([ns.map(entry).join('')], 'dev.mbox', { type: 'application/mbox' })
|
||||
|
||||
it('uploads the file, shows what happened and lists the new messages', async () => {
|
||||
const { user, state } = await setup([])
|
||||
await screen.findByText(/no messages archived yet/i)
|
||||
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1, 2, 3))
|
||||
|
||||
const status = await screen.findByRole('status')
|
||||
expect(status).toHaveTextContent('Imported 3 messages')
|
||||
expect(state.importTypes).toEqual(['application/mbox'])
|
||||
expect(state.archive[1]).toHaveLength(3)
|
||||
expect(await screen.findByRole('button', { name: 'Imported 3' })).toBeInTheDocument()
|
||||
expect(screen.getByRole('heading', { name: 'Archive (3)', level: 2 })).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('says when messages were already there', async () => {
|
||||
const { user } = await setup([])
|
||||
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1, 2))
|
||||
await screen.findByText(/Imported 2 messages/)
|
||||
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1, 2, 3))
|
||||
const status = await screen.findByText(/Imported 1 message\b/)
|
||||
expect(status).toHaveTextContent('2 already present')
|
||||
})
|
||||
|
||||
it('lists the messages that could not be read', async () => {
|
||||
const { user } = await setup([])
|
||||
const file = new File([entry(1) + 'From x@y.z Tue Jan 2 15:04:05 2024\n\nno headers here\n'], 'bad.mbox')
|
||||
await user.upload(screen.getByLabelText(/mbox file/i), file)
|
||||
const status = await screen.findByRole('status')
|
||||
expect(status).toHaveTextContent('1 failed')
|
||||
expect(within(status).getByText(/message 2: parse message: no headers/)).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('shows a server error', async () => {
|
||||
const { user, state } = await setup([])
|
||||
state.fail['POST /lists/1/archive/import'] = { status: 500, error: 'disk full' }
|
||||
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1))
|
||||
expect(await screen.findByRole('alert')).toHaveTextContent('disk full')
|
||||
expect(screen.queryByRole('status')).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('shows that it is working, and cannot be started twice', async () => {
|
||||
let release!: () => void
|
||||
const { user, state } = await setup([])
|
||||
state.hold = new Promise<void>((resolve) => (release = resolve))
|
||||
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1))
|
||||
const busy = await screen.findByRole('button', { name: /importing/i })
|
||||
expect(busy).toBeDisabled()
|
||||
release()
|
||||
expect(await screen.findByRole('status')).toHaveTextContent('Imported 1 message')
|
||||
expect(screen.getByRole('button', { name: /^import mbox$/i })).toBeEnabled()
|
||||
})
|
||||
|
||||
it('opens the file chooser from the button and accepts mbox-like files', async () => {
|
||||
await setup([])
|
||||
const input = screen.getByLabelText(/mbox file/i) as HTMLInputElement
|
||||
expect(input.type).toBe('file')
|
||||
expect(input.accept).toMatch(/\.mbox/)
|
||||
expect(screen.getByRole('button', { name: /^import mbox$/i })).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('can import the same file again after a first attempt (the input is reset)', async () => {
|
||||
const { user, state } = await setup([])
|
||||
const file = mboxFile(1)
|
||||
await user.upload(screen.getByLabelText(/mbox file/i), file)
|
||||
await screen.findByText(/Imported 1 message/)
|
||||
await user.upload(screen.getByLabelText(/mbox file/i), file)
|
||||
await waitFor(() => expect(state.importTypes).toHaveLength(2))
|
||||
})
|
||||
|
||||
it('forgets the previous result when a new import starts', async () => {
|
||||
const { user, state } = await setup([])
|
||||
state.fail['POST /lists/1/archive/import'] = { status: 500, error: 'disk full' }
|
||||
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1))
|
||||
expect(await screen.findByRole('alert')).toHaveTextContent('disk full')
|
||||
|
||||
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1))
|
||||
expect(await screen.findByRole('status')).toHaveTextContent('Imported 1 message')
|
||||
expect(screen.queryByRole('alert')).not.toBeInTheDocument() // the old error is gone
|
||||
|
||||
state.fail['POST /lists/1/archive/import'] = { status: 500, error: 'again' }
|
||||
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(2))
|
||||
expect(await screen.findByRole('alert')).toHaveTextContent('again')
|
||||
expect(screen.queryByRole('status')).not.toBeInTheDocument() // the old summary is gone
|
||||
})
|
||||
|
||||
it('says how many more failures there are than the server listed', async () => {
|
||||
const { user } = await setup([])
|
||||
const real = fetch
|
||||
vi.stubGlobal('fetch', vi.fn((url: string, init?: RequestInit) => {
|
||||
if (url.endsWith('/archive/import')) {
|
||||
const errors = Array.from({ length: 20 }, (_, i) => `message ${i + 1}: parse message: no headers`)
|
||||
return Promise.resolve(new Response(JSON.stringify({ imported: 0, duplicates: 0, failed: 25, errors }), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
}))
|
||||
}
|
||||
return real(url, init)
|
||||
}))
|
||||
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1))
|
||||
const status = await screen.findByRole('status')
|
||||
expect(status).toHaveTextContent('25 failed')
|
||||
expect(within(status).getByText(/and 5 more/)).toBeInTheDocument()
|
||||
expect(within(status).getAllByRole('listitem')).toHaveLength(21)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,22 +1,48 @@
|
|||
import { keepPreviousData, useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
|
||||
import { useState, type FormEvent } from 'react'
|
||||
import { useRef, useState, type ChangeEvent, type FormEvent } from 'react'
|
||||
import { useOutletContext } from 'react-router'
|
||||
import { toast } from 'sonner'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from '@/components/ui/dialog'
|
||||
import { Input } from '@/components/ui/input'
|
||||
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table'
|
||||
import { api, type List } from '@/lib/api'
|
||||
import { api, type ImportSummary, type List } from '@/lib/api'
|
||||
import { formatBytes, formatDate } from '@/lib/format'
|
||||
|
||||
const PAGE_SIZE = 25
|
||||
|
||||
function plural(n: number, one: string, many: string) {
|
||||
return `${n} ${n === 1 ? one : many}`
|
||||
}
|
||||
|
||||
export function ArchiveTab() {
|
||||
const list = useOutletContext<List>()
|
||||
const [input, setInput] = useState('')
|
||||
const [q, setQ] = useState('')
|
||||
const [page, setPage] = useState(0)
|
||||
const [openId, setOpenId] = useState<number | null>(null)
|
||||
const queryClient = useQueryClient()
|
||||
const fileInput = useRef<HTMLInputElement>(null)
|
||||
const [imported, setImported] = useState<ImportSummary | null>(null)
|
||||
const [importError, setImportError] = useState<string | null>(null)
|
||||
|
||||
const importMbox = useMutation({
|
||||
mutationFn: (file: File) => api.archive.import(list.id, file),
|
||||
onSuccess: async (summary) => {
|
||||
setImported(summary)
|
||||
await queryClient.invalidateQueries({ queryKey: ['archive', list.id] })
|
||||
},
|
||||
onError: (e: Error) => setImportError(e.message),
|
||||
})
|
||||
|
||||
function chooseFile(e: ChangeEvent<HTMLInputElement>) {
|
||||
const file = e.target.files?.[0]
|
||||
e.target.value = '' // lets the same file be chosen again
|
||||
if (!file) return
|
||||
setImported(null)
|
||||
setImportError(null)
|
||||
importMbox.mutate(file)
|
||||
}
|
||||
|
||||
const archive = useQuery({
|
||||
queryKey: ['archive', list.id, q, page],
|
||||
|
|
@ -43,7 +69,42 @@ export function ArchiveTab() {
|
|||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<h2 className="text-lg font-semibold">Archive{archive.data && !q ? ` (${total})` : ''}</h2>
|
||||
<div className="flex items-center justify-between">
|
||||
<h2 className="text-lg font-semibold">Archive{archive.data && !q ? ` (${total})` : ''}</h2>
|
||||
<input
|
||||
ref={fileInput}
|
||||
type="file"
|
||||
accept=".mbox,.mbx,.txt,application/mbox,text/plain"
|
||||
aria-label="Mbox file"
|
||||
className="hidden"
|
||||
onChange={chooseFile}
|
||||
/>
|
||||
<Button variant="outline" disabled={importMbox.isPending} onClick={() => fileInput.current?.click()}>
|
||||
{importMbox.isPending ? 'Importing…' : 'Import mbox'}
|
||||
</Button>
|
||||
</div>
|
||||
{importError && (
|
||||
<p role="alert" className="text-sm text-destructive">
|
||||
{importError}
|
||||
</p>
|
||||
)}
|
||||
{imported && (
|
||||
<div role="status" className="space-y-1 rounded-md border p-3 text-sm">
|
||||
<p>
|
||||
Imported {plural(imported.imported, 'message', 'messages')}
|
||||
{imported.duplicates > 0 && `, ${imported.duplicates} already present`}
|
||||
{imported.failed > 0 && `, ${imported.failed} failed`}
|
||||
</p>
|
||||
{imported.errors.length > 0 && (
|
||||
<ul className="list-disc pl-5 text-destructive">
|
||||
{imported.errors.map((e) => (
|
||||
<li key={e}>{e}</li>
|
||||
))}
|
||||
{imported.failed > imported.errors.length && <li>… and {imported.failed - imported.errors.length} more</li>}
|
||||
</ul>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{list.archive_policy === 'off' && (
|
||||
<p className="rounded-md border p-3 text-sm text-muted-foreground">
|
||||
Archiving is off for this list: new messages are not stored. Set Archive to private or public in Settings.
|
||||
|
|
|
|||
|
|
@ -11,6 +11,10 @@ export interface FakeState {
|
|||
queue: QueueItem[]
|
||||
/** Like the server, the queue listing is capped. */
|
||||
queueLimit: number
|
||||
/** Content types seen on archive imports. */
|
||||
importTypes: string[]
|
||||
/** When set, archive imports wait for it (to look at the busy state). */
|
||||
hold?: Promise<void>
|
||||
nextId: number
|
||||
/** Every request seen, as "METHOD /path". */
|
||||
calls: string[]
|
||||
|
|
@ -19,7 +23,7 @@ export interface FakeState {
|
|||
}
|
||||
|
||||
export function newState(overrides: Partial<FakeState> = {}): FakeState {
|
||||
return { token: 'good', lists: [], members: {}, held: {}, archive: {}, queue: [], queueLimit: 100, nextId: 100, calls: [], fail: {}, ...overrides }
|
||||
return { token: 'good', lists: [], members: {}, held: {}, archive: {}, queue: [], queueLimit: 100, importTypes: [], nextId: 100, calls: [], fail: {}, ...overrides }
|
||||
}
|
||||
|
||||
export function aList(over: Partial<List> = {}): List {
|
||||
|
|
@ -62,7 +66,8 @@ export function installFakeApi(state: FakeState) {
|
|||
delete state.fail[key]
|
||||
return err(f.status, f.error)
|
||||
}
|
||||
const body = init.body ? JSON.parse(init.body as string) : {}
|
||||
// Files (archive imports) are not JSON.
|
||||
const body = typeof init.body === 'string' && init.body ? JSON.parse(init.body) : {}
|
||||
let m: RegExpMatchArray | null
|
||||
|
||||
if (path === '/queue' && method === 'GET') {
|
||||
|
|
@ -132,6 +137,34 @@ export function installFakeApi(state: FakeState) {
|
|||
}
|
||||
return err(404, 'not found')
|
||||
}
|
||||
if ((m = path.match(/^\/lists\/(\d+)\/archive\/import$/)) && method === 'POST') {
|
||||
const id = Number(m[1])
|
||||
if (!state.lists.some((l) => l.id === id)) return err(404, 'not found')
|
||||
if (state.hold) await state.hold
|
||||
state.importTypes.push((init.headers as Record<string, string>)['Content-Type'] ?? '')
|
||||
const text = typeof init.body === 'string' ? init.body : await (init.body as Blob).text()
|
||||
const messages = text.split(/^From \S+ .*$/m).slice(1)
|
||||
const archive = (state.archive[id] ??= [])
|
||||
let imported = 0
|
||||
let duplicates = 0
|
||||
const errors: string[] = []
|
||||
messages.forEach((raw, i) => {
|
||||
const subject = /^Subject: (.*)$/m.exec(raw)?.[1]
|
||||
const messageId = /^Message-ID: (.*)$/m.exec(raw)?.[1]
|
||||
if (!subject || !messageId) {
|
||||
errors.push(`message ${i + 1}: parse message: no headers`)
|
||||
return
|
||||
}
|
||||
if (archive.some((a) => a.message_id === messageId)) {
|
||||
duplicates++
|
||||
return
|
||||
}
|
||||
const nid = state.nextId++
|
||||
archive.push({ id: nid, thread_id: nid, message_id: messageId, subject, sender: 'Imported', date: '2024-01-01T00:00:00Z', size: raw.length })
|
||||
imported++
|
||||
})
|
||||
return json(200, { imported, duplicates, failed: errors.length, errors })
|
||||
}
|
||||
if ((m = path.match(/^\/lists\/(\d+)\/archive$/)) && method === 'GET') {
|
||||
const id = Number(m[1])
|
||||
const q = (url.searchParams.get('q') ?? '').toLowerCase()
|
||||
|
|
|
|||
|
|
@ -275,3 +275,32 @@ paths:
|
|||
responses:
|
||||
'204': {description: Discarded}
|
||||
'404': {$ref: '#/components/responses/Error'}
|
||||
/lists/{id}/archive/import:
|
||||
parameters:
|
||||
- {name: id, in: path, required: true, schema: {type: integer}}
|
||||
post:
|
||||
summary: Import an mbox file into the archive
|
||||
description: >
|
||||
The request body is the mbox file itself (any content type, `application/mbox` by convention), streamed.
|
||||
Idempotent: a message already archived (same Message-ID) is counted as a duplicate. Threads are rebuilt at the end.
|
||||
Works whatever the list's archive policy. Messages that cannot be read are reported in the summary and do not
|
||||
fail the request; a read error while receiving the body does (400), keeping the messages read so far.
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/mbox:
|
||||
schema: {type: string, format: binary}
|
||||
responses:
|
||||
'200':
|
||||
description: Summary
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
imported: {type: integer}
|
||||
duplicates: {type: integer}
|
||||
failed: {type: integer}
|
||||
errors: {type: array, items: {type: string}, description: The first 20 failures}
|
||||
'400': {$ref: '#/components/responses/Error'}
|
||||
'404': {$ref: '#/components/responses/Error'}
|
||||
|
|
|
|||
|
|
@ -544,6 +544,70 @@ func TestWholePath(t *testing.T) {
|
|||
}
|
||||
})
|
||||
|
||||
t.Run("a Mailman-style mbox can be imported and browsed", func(t *testing.T) {
|
||||
mbox := "From MAILER-DAEMON Mon Jan 1 10:00:00 2024\n" +
|
||||
"From: =?UTF-8?Q?Jos=C3=A9_Garc=C3=ADa?= <jose@old-list.example>\nSubject: =?UTF-8?Q?Caff=C3=A8_a_las_nueve?=\nMessage-ID: <old-1@old-list.example>\nDate: Mon, 01 Jan 2024 10:00:00 +0000\nMIME-Version: 1.0\nContent-Type: multipart/alternative; boundary=BB\n\n" +
|
||||
"--BB\nContent-Type: text/plain; charset=utf-8\nContent-Transfer-Encoding: quoted-printable\n\nCiao a tutti, scrivetemi a jose@old-list.example\n>From the old archive: caff=C3=A8!\n--BB\nContent-Type: text/html\n\n<p>html copy</p>\n--BB--\n\n" +
|
||||
"From carol@members.test Tue Jan 2 09:00:00 2024\n" +
|
||||
"From: Carol <carol@members.test>\nSubject: Re: Caff\xc3\xa8\nMessage-ID: <old-0@old-list.example>\nIn-Reply-To: <old-1@old-list.example>\nDate: Tue, 02 Jan 2024 09:00:00 +0000\n\nsee you there\n"
|
||||
imp := exec.Command("docker", "exec", "-i", parsimonyC, "/parsimony", "archive", "import", list, "-")
|
||||
imp.Stdin = strings.NewReader(mbox)
|
||||
out, err := imp.CombinedOutput()
|
||||
if err != nil || !strings.Contains(string(out), "Imported 2 messages") {
|
||||
t.Fatalf("archive import: %v\n%s", err, out)
|
||||
}
|
||||
// Importing again adds nothing.
|
||||
again := exec.Command("docker", "exec", "-i", parsimonyC, "/parsimony", "archive", "import", list, "-")
|
||||
again.Stdin = strings.NewReader(mbox)
|
||||
out, err = again.CombinedOutput()
|
||||
if err != nil || !strings.Contains(string(out), "Imported 0 messages") || !strings.Contains(string(out), "2 already present") {
|
||||
t.Fatalf("second import: %v\n%s", err, out)
|
||||
}
|
||||
|
||||
var page struct {
|
||||
Messages []struct {
|
||||
ID int64 `json:"id"`
|
||||
ThreadID int64 `json:"thread_id"`
|
||||
Subject string `json:"subject"`
|
||||
Sender string `json:"sender"`
|
||||
} `json:"messages"`
|
||||
}
|
||||
mustCall(t, 200, "GET", "/api/v1/lists/1/archive?q=nueve", "", &page)
|
||||
if len(page.Messages) != 1 || page.Messages[0].Subject != "Caffè a las nueve" || page.Messages[0].Sender != "José García" {
|
||||
t.Fatalf("encoded headers must be decoded: %+v", page.Messages)
|
||||
}
|
||||
root := page.Messages[0]
|
||||
var msg struct {
|
||||
Body string `json:"body"`
|
||||
}
|
||||
mustCall(t, 200, "GET", fmt.Sprintf("/api/v1/archive/%d", root.ID), "", &msg)
|
||||
if !strings.Contains(msg.Body, "caffè!") || !strings.Contains(msg.Body, "\nFrom the old archive") || strings.Contains(msg.Body, "html copy") {
|
||||
t.Fatalf("body: %q", msg.Body)
|
||||
}
|
||||
var all struct {
|
||||
Messages []struct {
|
||||
ThreadID int64 `json:"thread_id"`
|
||||
Subject string `json:"subject"`
|
||||
} `json:"messages"`
|
||||
}
|
||||
mustCall(t, 200, "GET", "/api/v1/lists/1/archive?q=caffe", "", &all)
|
||||
if len(all.Messages) != 2 || all.Messages[0].ThreadID != all.Messages[1].ThreadID {
|
||||
t.Fatalf("the reply must be in its parent's thread: %+v", all.Messages)
|
||||
}
|
||||
|
||||
// The list's archive is public: imported messages show up on the web, with addresses hidden.
|
||||
resp, err := http.Get(fmt.Sprintf("%s/archive/%s/%d", webBase, list, root.ID))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != 200 || !strings.Contains(string(b), "José García") || !strings.Contains(string(b), "jose@…") ||
|
||||
strings.Contains(string(b), "jose@old-list.example") {
|
||||
t.Fatalf("public page: %d\n%.700s", resp.StatusCode, b)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("the queue ends empty", func(t *testing.T) {
|
||||
waitQueueEmpty(t)
|
||||
})
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import (
|
|||
"crypto/subtle"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/mail"
|
||||
|
|
@ -14,6 +15,7 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"codeberg.org/squidd/parsimony/internal/archive"
|
||||
"codeberg.org/squidd/parsimony/internal/store"
|
||||
)
|
||||
|
||||
|
|
@ -46,6 +48,7 @@ func New(s *store.Store, token string, rel Releaser) http.Handler {
|
|||
mux.Handle("POST /api/v1/lists/{id}/members", srv.auth(srv.addMember))
|
||||
mux.Handle("DELETE /api/v1/lists/{id}/members/{address}", srv.auth(srv.removeMember))
|
||||
mux.Handle("GET /api/v1/lists/{id}/archive", srv.auth(srv.listArchive))
|
||||
mux.Handle("POST /api/v1/lists/{id}/archive/import", srv.auth(srv.importArchive))
|
||||
mux.Handle("GET /api/v1/archive/{id}", srv.auth(srv.getArchived))
|
||||
mux.Handle("DELETE /api/v1/archive/{id}", srv.auth(srv.deleteArchived))
|
||||
mux.Handle("GET /api/v1/queue", srv.auth(srv.getQueue))
|
||||
|
|
@ -543,3 +546,22 @@ func (s *server) discardQueued(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// importArchive reads an mbox from the request body, streaming, and archives it.
|
||||
// Failed messages are reported in the summary; the request itself succeeds.
|
||||
func (s *server) importArchive(w http.ResponseWriter, r *http.Request) {
|
||||
id, ok := listID(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if _, err := s.store.GetList(id); err != nil {
|
||||
storeError(w, err)
|
||||
return
|
||||
}
|
||||
sum, err := archive.ImportMbox(s.store, id, r.Body, time.Now())
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, fmt.Sprintf("import stopped after %d message(s): %v", sum.Imported, err))
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, sum)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package api
|
|||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
|
|
@ -522,3 +523,109 @@ func TestQueueRequiresAuth(t *testing.T) {
|
|||
expect(t, do(t, h, "GET", "/api/v1/queue", "", false), 401)
|
||||
expect(t, do(t, h, "DELETE", "/api/v1/queue/1", "", false), 401)
|
||||
}
|
||||
|
||||
const importMbox = "From alice@example.com Tue Jan 2 15:04:05 2024\nFrom: Alice <alice@example.com>\nSubject: Kickoff\nMessage-ID: <1@x>\nDate: Mon, 01 Jan 2024 10:00:00 +0000\n\nlet's start\n\n" +
|
||||
"From bob@example.com Tue Jan 2 15:04:05 2024\nFrom: Bob <bob@example.com>\nSubject: Re: Kickoff\nMessage-ID: <2@x>\nIn-Reply-To: <1@x>\nDate: Mon, 01 Jan 2024 11:00:00 +0000\n\nagreed\n"
|
||||
|
||||
type importJSON struct {
|
||||
Imported int `json:"imported"`
|
||||
Duplicates int `json:"duplicates"`
|
||||
Failed int `json:"failed"`
|
||||
Errors []string `json:"errors"`
|
||||
}
|
||||
|
||||
func TestArchiveImport(t *testing.T) {
|
||||
h, s := queueServer(t)
|
||||
s.CreateList("dev", "example.org", "Dev")
|
||||
|
||||
rec := do(t, h, "POST", "/api/v1/lists/1/archive/import", importMbox, true)
|
||||
expect(t, rec, 200)
|
||||
var sum importJSON
|
||||
decode(t, rec, &sum)
|
||||
if sum.Imported != 2 || sum.Duplicates != 0 || sum.Failed != 0 || sum.Errors == nil || len(sum.Errors) != 0 {
|
||||
t.Fatalf("summary: %+v (errors must be [] not null)", sum)
|
||||
}
|
||||
var page archiveJSON
|
||||
decode(t, do(t, h, "GET", "/api/v1/lists/1/archive", "", true), &page)
|
||||
if page.Total != 2 || page.Messages[0].Subject != "Re: Kickoff" {
|
||||
t.Fatalf("archive: %+v", page)
|
||||
}
|
||||
if page.Messages[0].ThreadID != page.Messages[1].ThreadID {
|
||||
t.Fatal("imported replies must be threaded")
|
||||
}
|
||||
|
||||
rec = do(t, h, "POST", "/api/v1/lists/1/archive/import", importMbox, true)
|
||||
expect(t, rec, 200)
|
||||
decode(t, rec, &sum)
|
||||
if sum.Imported != 0 || sum.Duplicates != 2 {
|
||||
t.Fatalf("second import: %+v", sum)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveImportReportsFailuresWithoutFailingTheRequest(t *testing.T) {
|
||||
h, s := queueServer(t)
|
||||
s.CreateList("dev", "example.org", "Dev")
|
||||
body := importMbox + "\nFrom x@y.z Tue Jan 2 15:04:05 2024\n\nno headers\n"
|
||||
rec := do(t, h, "POST", "/api/v1/lists/1/archive/import", body, true)
|
||||
expect(t, rec, 200)
|
||||
var sum importJSON
|
||||
decode(t, rec, &sum)
|
||||
if sum.Imported != 2 || sum.Failed != 1 || len(sum.Errors) != 1 || !strings.Contains(sum.Errors[0], "message 3") {
|
||||
t.Fatalf("summary: %+v", sum)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveImportEdgeCases(t *testing.T) {
|
||||
h, s := queueServer(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
s.SetPolicies(l.ID, store.PostMembers, store.JoinOpen, store.ArchiveOff)
|
||||
expect(t, do(t, h, "POST", "/api/v1/lists/99/archive/import", importMbox, true), 404)
|
||||
expect(t, do(t, h, "POST", "/api/v1/lists/abc/archive/import", importMbox, true), 404)
|
||||
expect(t, do(t, h, "POST", "/api/v1/lists/1/archive/import", importMbox, false), 401)
|
||||
|
||||
rec := do(t, h, "POST", "/api/v1/lists/1/archive/import", "", true)
|
||||
expect(t, rec, 200)
|
||||
var sum importJSON
|
||||
decode(t, rec, &sum)
|
||||
if sum.Imported != 0 || sum.Failed != 0 {
|
||||
t.Fatalf("empty body: %+v", sum)
|
||||
}
|
||||
rec = do(t, h, "POST", "/api/v1/lists/1/archive/import", importMbox, true)
|
||||
expect(t, rec, 200)
|
||||
decode(t, rec, &sum)
|
||||
if sum.Imported != 2 {
|
||||
t.Fatalf("an import works even when the archive policy is off: %+v", sum)
|
||||
}
|
||||
}
|
||||
|
||||
type failingBody struct {
|
||||
data string
|
||||
done bool
|
||||
}
|
||||
|
||||
func (f *failingBody) Read(p []byte) (int, error) {
|
||||
if !f.done {
|
||||
f.done = true
|
||||
return copy(p, f.data), nil
|
||||
}
|
||||
return 0, errors.New("connection reset by peer")
|
||||
}
|
||||
|
||||
func TestArchiveImportStoppedByAReadErrorIsAnErrorResponse(t *testing.T) {
|
||||
h, s := queueServer(t)
|
||||
s.CreateList("dev", "example.org", "Dev")
|
||||
req := httptest.NewRequest("POST", "/api/v1/lists/1/archive/import", &failingBody{data: importMbox})
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, req)
|
||||
expect(t, rec, 400)
|
||||
var e struct{ Error string }
|
||||
decode(t, rec, &e)
|
||||
if !strings.Contains(e.Error, "connection reset") || !strings.Contains(e.Error, "1 message") {
|
||||
t.Fatalf("the error says what happened and how far it got: %q", e.Error)
|
||||
}
|
||||
l, _ := s.FindList("dev@example.org")
|
||||
if _, total, _ := s.ListMessages(l.ID, 10, 0); total != 1 {
|
||||
t.Fatalf("the first message was complete and is kept, got %d", total)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
106
internal/archive/import.go
Normal file
106
internal/archive/import.go
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
package archive
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"codeberg.org/squidd/parsimony/internal/mbox"
|
||||
"codeberg.org/squidd/parsimony/internal/store"
|
||||
)
|
||||
|
||||
const (
|
||||
// MaxImportMessage is the largest message accepted from an mbox, in bytes.
|
||||
MaxImportMessage = 25 << 20
|
||||
// MaxReportedErrors caps the error messages kept in a summary; Failed still counts every one.
|
||||
MaxReportedErrors = 20
|
||||
|
||||
importBatch = 200 // messages per transaction
|
||||
)
|
||||
|
||||
// ImportSummary reports what an import did.
|
||||
type ImportSummary struct {
|
||||
Imported int `json:"imported"` // new messages stored
|
||||
Duplicates int `json:"duplicates"` // already in the archive (same Message-ID)
|
||||
Failed int `json:"failed"` // unreadable or too large
|
||||
Errors []string `json:"errors"` // the first MaxReportedErrors failures
|
||||
}
|
||||
|
||||
// ImportMbox archives every message of an mbox into a list. It is idempotent
|
||||
// (a message is identified by its Message-ID) and works whatever the list's
|
||||
// archive policy is: the policy only governs live mail. Threads are rebuilt at
|
||||
// the end, so the order of the messages in the file does not matter. now is the
|
||||
// date of last resort for a message with neither a Date header nor a dated
|
||||
// "From " line. On a read error the messages read so far are kept and returned
|
||||
// in the summary together with the error.
|
||||
func ImportMbox(st *store.Store, listID int64, r io.Reader, now time.Time) (ImportSummary, error) {
|
||||
sum := ImportSummary{Errors: []string{}}
|
||||
var batch []store.Message
|
||||
|
||||
fail := func(index int, err error) {
|
||||
sum.Failed++
|
||||
if len(sum.Errors) < MaxReportedErrors {
|
||||
sum.Errors = append(sum.Errors, fmt.Sprintf("message %d: %v", index, err))
|
||||
}
|
||||
}
|
||||
flush := func() error {
|
||||
results, err := st.ArchiveBatch(listID, batch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, res := range results {
|
||||
if res.Created {
|
||||
sum.Imported++
|
||||
} else {
|
||||
sum.Duplicates++
|
||||
}
|
||||
}
|
||||
batch = batch[:0]
|
||||
return nil
|
||||
}
|
||||
finish := func(readErr error) (ImportSummary, error) {
|
||||
if err := flush(); err != nil {
|
||||
return sum, err
|
||||
}
|
||||
if sum.Imported > 0 {
|
||||
if _, err := st.Rethread(listID); err != nil {
|
||||
return sum, err
|
||||
}
|
||||
}
|
||||
return sum, readErr
|
||||
}
|
||||
|
||||
rd := mbox.NewReader(r, MaxImportMessage)
|
||||
for {
|
||||
m, err := rd.Next()
|
||||
if err == io.EOF {
|
||||
return finish(nil)
|
||||
}
|
||||
if errors.Is(err, mbox.ErrTooLarge) {
|
||||
fail(m.Index, fmt.Errorf("too large (limit %d MiB)", MaxImportMessage>>20))
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return finish(fmt.Errorf("read mbox: %w", err))
|
||||
}
|
||||
fallback := now
|
||||
if d, ok := m.Date(); ok {
|
||||
fallback = d
|
||||
}
|
||||
en, err := Parse(m.Data, fallback)
|
||||
if err != nil {
|
||||
fail(m.Index, err)
|
||||
continue
|
||||
}
|
||||
batch = append(batch, store.Message{
|
||||
MessageID: en.MessageID, InReplyTo: en.InReplyTo, Subject: en.Subject,
|
||||
Sender: en.Sender, Date: en.Date, Body: en.Body,
|
||||
})
|
||||
if len(batch) >= importBatch {
|
||||
if err := flush(); err != nil {
|
||||
return sum, err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
254
internal/archive/import_test.go
Normal file
254
internal/archive/import_test.go
Normal file
|
|
@ -0,0 +1,254 @@
|
|||
package archive
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"codeberg.org/squidd/parsimony/internal/store"
|
||||
)
|
||||
|
||||
var importNow = time.Date(2031, 5, 6, 7, 8, 9, 0, time.UTC)
|
||||
|
||||
func newList(t *testing.T) (*store.Store, store.List) {
|
||||
t.Helper()
|
||||
s, err := store.Open(":memory:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { s.Close() })
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
return s, l
|
||||
}
|
||||
|
||||
func mboxMsg(from, subject, id, replyTo, date, body string) string {
|
||||
addr := from
|
||||
if i := strings.Index(from, "<"); i >= 0 {
|
||||
addr = strings.Trim(from[i:], "<>")
|
||||
}
|
||||
h := "From " + addr + " Tue Jan 2 15:04:05 2024\n"
|
||||
h += "From: " + from + "\nSubject: " + subject + "\n"
|
||||
if id != "" {
|
||||
h += "Message-ID: " + id + "\n"
|
||||
}
|
||||
if replyTo != "" {
|
||||
h += "In-Reply-To: " + replyTo + "\n"
|
||||
}
|
||||
if date != "" {
|
||||
h += "Date: " + date + "\n"
|
||||
}
|
||||
return h + "\n" + body + "\n\n"
|
||||
}
|
||||
|
||||
func TestImportMbox(t *testing.T) {
|
||||
s, l := newList(t)
|
||||
mbox := mboxMsg("Alice <alice@example.com>", "Kickoff", "<1@x>", "", "Mon, 01 Jan 2024 10:00:00 +0000", "let's start") +
|
||||
mboxMsg("Bob <bob@example.com>", "Re: Kickoff", "<2@x>", "<1@x>", "Mon, 01 Jan 2024 11:00:00 +0000", "agreed") +
|
||||
mboxMsg("Carol <carol@example.com>", "Lunch", "<3@x>", "", "Tue, 02 Jan 2024 12:00:00 +0000", "pizza?")
|
||||
sum, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sum.Imported != 3 || sum.Duplicates != 0 || sum.Failed != 0 || len(sum.Errors) != 0 {
|
||||
t.Fatalf("summary: %+v", sum)
|
||||
}
|
||||
ms, total, _ := s.ListMessages(l.ID, 10, 0)
|
||||
if total != 3 || ms[0].Subject != "Lunch" {
|
||||
t.Fatalf("messages: %+v", ms)
|
||||
}
|
||||
var reply store.Message
|
||||
for _, m := range ms {
|
||||
if m.Subject == "Re: Kickoff" {
|
||||
reply, _ = s.GetMessage(m.ID)
|
||||
}
|
||||
}
|
||||
if reply.Sender != "Bob" || reply.Body != "agreed\n" || !reply.Date.Equal(time.Date(2024, 1, 1, 11, 0, 0, 0, time.UTC)) {
|
||||
t.Fatalf("reply: %+v", reply)
|
||||
}
|
||||
root, _ := s.GetMessage(reply.ThreadID)
|
||||
if root.Subject != "Kickoff" {
|
||||
t.Fatalf("the reply must be threaded under its parent, got %q", root.Subject)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportIsIdempotent(t *testing.T) {
|
||||
s, l := newList(t)
|
||||
mbox := mboxMsg("A <a@x.com>", "One", "<1@x>", "", "", "b1") + mboxMsg("B <b@x.com>", "Two", "", "", "", "b2")
|
||||
if sum, _ := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow); sum.Imported != 2 {
|
||||
t.Fatalf("first: %+v", sum)
|
||||
}
|
||||
sum, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow)
|
||||
if err != nil || sum.Imported != 0 || sum.Duplicates != 2 || sum.Failed != 0 {
|
||||
t.Fatalf("second import must add nothing, even for a message without a Message-ID: %+v %v", sum, err)
|
||||
}
|
||||
if _, total, _ := s.ListMessages(l.ID, 10, 0); total != 2 {
|
||||
t.Fatalf("total = %d", total)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportRebuildsThreadsWhateverTheOrder(t *testing.T) {
|
||||
s, l := newList(t)
|
||||
mbox := mboxMsg("C <c@x.com>", "Re: Re: T", "<c@x>", "<b@x>", "", "c") +
|
||||
mboxMsg("B <b@x.com>", "Re: T", "<b@x>", "<a@x>", "", "b") +
|
||||
mboxMsg("A <a@x.com>", "T", "<a@x>", "", "", "a")
|
||||
if _, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ms, _, _ := s.ListMessages(l.ID, 10, 0)
|
||||
threads := map[int64]int{}
|
||||
for _, m := range ms {
|
||||
threads[m.ThreadID]++
|
||||
}
|
||||
if len(threads) != 1 {
|
||||
t.Fatalf("one thread expected, got %v", threads)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportReportsBadMessagesAndKeepsGoing(t *testing.T) {
|
||||
s, l := newList(t)
|
||||
mbox := mboxMsg("A <a@x.com>", "Good one", "<1@x>", "", "", "ok") +
|
||||
"From x@y.z Tue Jan 2 15:04:05 2024\n\n\nthis message has no headers at all\n\n" +
|
||||
mboxMsg("B <b@x.com>", "Good two", "<2@x>", "", "", "ok")
|
||||
sum, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sum.Imported != 2 || sum.Failed != 1 || len(sum.Errors) != 1 || !strings.Contains(sum.Errors[0], "message 2") {
|
||||
t.Fatalf("summary: %+v", sum)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportSkipsOversizedMessages(t *testing.T) {
|
||||
s, l := newList(t)
|
||||
big := strings.Repeat("0123456789", MaxImportMessage/10+10)
|
||||
mbox := mboxMsg("A <a@x.com>", "Small", "<1@x>", "", "", "ok") + mboxMsg("B <b@x.com>", "Huge", "<2@x>", "", "", big) +
|
||||
mboxMsg("C <c@x.com>", "Small too", "<3@x>", "", "", "ok")
|
||||
sum, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sum.Imported != 2 || sum.Failed != 1 || !strings.Contains(sum.Errors[0], "message 2") || !strings.Contains(sum.Errors[0], "too large") {
|
||||
t.Fatalf("summary: %+v", sum)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportDateFallbacks(t *testing.T) {
|
||||
s, l := newList(t)
|
||||
mbox := mboxMsg("A <a@x.com>", "has Date", "<1@x>", "", "Mon, 01 Jan 2024 10:00:00 +0000", "b") +
|
||||
mboxMsg("B <b@x.com>", "from envelope", "<2@x>", "", "", "b") +
|
||||
"From c@x.com Fri Feb 23 08:30 2024\nFrom: C <c@x.com>\nSubject: nothing\nMessage-ID: <3@x>\n\nb\n\n"
|
||||
if _, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ms, _, _ := s.ListMessages(l.ID, 10, 0)
|
||||
dates := map[string]time.Time{}
|
||||
for _, m := range ms {
|
||||
dates[m.Subject] = m.Date
|
||||
}
|
||||
if !dates["has Date"].Equal(time.Date(2024, 1, 1, 10, 0, 0, 0, time.UTC)) {
|
||||
t.Errorf("Date header wins: %v", dates["has Date"])
|
||||
}
|
||||
if !dates["from envelope"].Equal(time.Date(2024, 1, 2, 15, 4, 5, 0, time.UTC)) {
|
||||
t.Errorf("the From line is the fallback: %v", dates["from envelope"])
|
||||
}
|
||||
if !dates["nothing"].Equal(time.Date(2024, 2, 23, 8, 30, 0, 0, time.UTC)) {
|
||||
t.Errorf("minute-precision envelope date: %v", dates["nothing"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportDateFallsBackToNowAsALastResort(t *testing.T) {
|
||||
s, l := newList(t)
|
||||
// A valid separator whose date has trailing junk, so it cannot be parsed, and no Date header.
|
||||
mbox := "From a@x.com Tue Jan 2 15:04:05 2024 extra junk after\nFrom: A <a@x.com>\nSubject: undated\nMessage-ID: <1@x>\n\nb\n"
|
||||
sum, _ := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow)
|
||||
if sum.Imported != 1 {
|
||||
t.Fatalf("%+v", sum)
|
||||
}
|
||||
ms, _, _ := s.ListMessages(l.ID, 10, 0)
|
||||
if !ms[0].Date.Equal(importNow) {
|
||||
t.Fatalf("date = %v, want the import time", ms[0].Date)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportCapsTheErrorListButCountsEverything(t *testing.T) {
|
||||
s, l := newList(t)
|
||||
var b strings.Builder
|
||||
for i := 0; i < 30; i++ {
|
||||
b.WriteString("From x@y.z Tue Jan 2 15:04:05 2024\n\nno headers here\n\n")
|
||||
}
|
||||
sum, err := ImportMbox(s, l.ID, strings.NewReader(b.String()), importNow)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sum.Failed != 30 || len(sum.Errors) != MaxReportedErrors {
|
||||
t.Fatalf("failed=%d errors=%d", sum.Failed, len(sum.Errors))
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportWorksWhateverTheArchivePolicy(t *testing.T) {
|
||||
s, l := newList(t)
|
||||
s.SetPolicies(l.ID, store.PostMembers, store.JoinOpen, store.ArchiveOff)
|
||||
sum, _ := ImportMbox(s, l.ID, strings.NewReader(mboxMsg("A <a@x.com>", "S", "<1@x>", "", "", "b")), importNow)
|
||||
if sum.Imported != 1 {
|
||||
t.Fatalf("an explicit import is not governed by the live-mail policy: %+v", sum)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportLargeFileCrossesBatchBoundaries(t *testing.T) {
|
||||
s, l := newList(t)
|
||||
var b strings.Builder
|
||||
n := importBatch*2 + 50
|
||||
for i := 0; i < n; i++ {
|
||||
parent := ""
|
||||
if i > 0 && i%7 != 0 {
|
||||
parent = fmt.Sprintf("<m%d@x>", i-1)
|
||||
}
|
||||
b.WriteString(mboxMsg("A <a@x.com>", fmt.Sprintf("Subject %d", i), fmt.Sprintf("<m%d@x>", i), parent, "", "body"))
|
||||
}
|
||||
sum, err := ImportMbox(s, l.ID, strings.NewReader(b.String()), importNow)
|
||||
if err != nil || sum.Imported != n || sum.Failed != 0 {
|
||||
t.Fatalf("summary: %+v %v", sum, err)
|
||||
}
|
||||
if _, total, _ := s.ListMessages(l.ID, 1, 0); total != n {
|
||||
t.Fatalf("stored %d of %d", total, n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportDoesNotTouchOtherLists(t *testing.T) {
|
||||
s, l := newList(t)
|
||||
other, _ := s.CreateList("ops", "example.org", "Ops")
|
||||
ImportMbox(s, l.ID, strings.NewReader(mboxMsg("A <a@x.com>", "S", "<1@x>", "", "", "b")), importNow)
|
||||
if _, total, _ := s.ListMessages(other.ID, 10, 0); total != 0 {
|
||||
t.Fatalf("other list has %d messages", total)
|
||||
}
|
||||
}
|
||||
|
||||
type brokenReader struct {
|
||||
data string
|
||||
done bool
|
||||
}
|
||||
|
||||
func (b *brokenReader) Read(p []byte) (int, error) {
|
||||
if !b.done {
|
||||
b.done = true
|
||||
return copy(p, b.data), nil
|
||||
}
|
||||
return 0, errors.New("connection reset")
|
||||
}
|
||||
|
||||
func TestImportReturnsTheProgressMadeBeforeAReadError(t *testing.T) {
|
||||
s, l := newList(t)
|
||||
data := mboxMsg("A <a@x.com>", "First", "<1@x>", "", "", "b") + mboxMsg("B <b@x.com>", "Second", "<2@x>", "", "", "b")
|
||||
sum, err := ImportMbox(s, l.ID, &brokenReader{data: data}, importNow)
|
||||
if err == nil || !strings.Contains(err.Error(), "connection reset") {
|
||||
t.Fatalf("want the read error, got %v", err)
|
||||
}
|
||||
if sum.Imported != 1 {
|
||||
t.Fatalf("the messages read before the failure are kept: %+v", sum)
|
||||
}
|
||||
if _, total, _ := s.ListMessages(l.ID, 10, 0); total != 1 {
|
||||
t.Fatalf("stored %d", total)
|
||||
}
|
||||
}
|
||||
|
|
@ -45,6 +45,8 @@ Administration (talks to the REST API):
|
|||
archive ls <list> [--q TEXT] [--limit N] [--offset N]
|
||||
archive show <id>
|
||||
archive rm <id> --yes
|
||||
archive import <list> <file|-> add the messages of an mbox file (Mailman export);
|
||||
safe to repeat, threads are rebuilt
|
||||
|
||||
queue ls outbound queue
|
||||
queue rm <id> discard one queued delivery
|
||||
|
|
@ -90,7 +92,7 @@ func Run(args []string, io IO, getenv func(string) string) int {
|
|||
"list": {"ls": c.listLs, "add": c.listAdd, "show": c.listShow, "set": c.listSet, "rm": c.listRm},
|
||||
"member": {"ls": c.memberLs, "add": c.memberAdd, "rm": c.memberRm, "import": c.memberImport},
|
||||
"held": {"ls": c.heldLs, "approve": c.heldApprove, "discard": c.heldDiscard},
|
||||
"archive": {"ls": c.archiveLs, "show": c.archiveShow, "rm": c.archiveRm},
|
||||
"archive": {"ls": c.archiveLs, "show": c.archiveShow, "rm": c.archiveRm, "import": c.archiveImport},
|
||||
"queue": {"ls": c.queueLs, "rm": c.queueRm},
|
||||
}
|
||||
group, ok := commands[rest[0]]
|
||||
|
|
@ -145,7 +147,11 @@ func (c *cmd) client() (*client, error) {
|
|||
if _, err := url.Parse(base); err != nil {
|
||||
return nil, fmt.Errorf("invalid URL %q", base)
|
||||
}
|
||||
c.cl = &client{base: strings.TrimRight(base, "/"), token: token, http: &http.Client{Timeout: 30 * time.Second}}
|
||||
c.cl = &client{
|
||||
base: strings.TrimRight(base, "/"), token: token,
|
||||
http: &http.Client{Timeout: 30 * time.Second},
|
||||
upload: &http.Client{}, // big files: no overall timeout
|
||||
}
|
||||
return c.cl, nil
|
||||
}
|
||||
|
||||
|
|
@ -176,7 +182,8 @@ func parseArgs(fs *flag.FlagSet, args []string) ([]string, error) {
|
|||
|
||||
type client struct {
|
||||
base, token string
|
||||
http *http.Client
|
||||
http *http.Client // ordinary requests, with a timeout
|
||||
upload *http.Client // streaming uploads, without one
|
||||
}
|
||||
|
||||
type apiError struct {
|
||||
|
|
@ -186,25 +193,35 @@ type apiError struct {
|
|||
|
||||
func (e *apiError) Error() string { return e.message }
|
||||
|
||||
// do sends a request and returns the raw response body (nil for 204).
|
||||
// do sends a JSON request and returns the raw response body (nil for 204).
|
||||
func (cl *client) do(method, path string, body any) (json.RawMessage, error) {
|
||||
var rd io.Reader
|
||||
ctype := ""
|
||||
if body != nil {
|
||||
b, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rd = bytes.NewReader(b)
|
||||
rd, ctype = bytes.NewReader(b), "application/json"
|
||||
}
|
||||
return cl.send(cl.http, method, path, rd, ctype)
|
||||
}
|
||||
|
||||
// uploadTo streams r as the body of a POST, with no overall timeout.
|
||||
func (cl *client) uploadTo(path string, r io.Reader, contentType string) (json.RawMessage, error) {
|
||||
return cl.send(cl.upload, "POST", path, r, contentType)
|
||||
}
|
||||
|
||||
func (cl *client) send(hc *http.Client, method, path string, rd io.Reader, contentType string) (json.RawMessage, error) {
|
||||
req, err := http.NewRequest(method, cl.base+"/api/v1"+path, rd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+cl.token)
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if contentType != "" {
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
}
|
||||
resp, err := cl.http.Do(req)
|
||||
resp, err := hc.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot reach %s: %w", cl.base, errors.Unwrap(err))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
|
|
@ -438,3 +439,116 @@ func TestQueue(t *testing.T) {
|
|||
t.Fatalf("pending = %d", st.Pending)
|
||||
}
|
||||
}
|
||||
|
||||
const cliMbox = "From alice@example.com Tue Jan 2 15:04:05 2024\nFrom: Alice <alice@example.com>\nSubject: Kickoff\nMessage-ID: <1@x>\nDate: Mon, 01 Jan 2024 10:00:00 +0000\n\nlet's start\n\n" +
|
||||
"From bob@example.com Tue Jan 2 15:04:05 2024\nFrom: Bob <bob@example.com>\nSubject: Re: Kickoff\nMessage-ID: <2@x>\nIn-Reply-To: <1@x>\nDate: Mon, 01 Jan 2024 11:00:00 +0000\n\nagreed\n"
|
||||
|
||||
func TestArchiveImportFromFileAndStdin(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
e.ok(t, "list", "add", "dev@example.org")
|
||||
path := t.TempDir() + "/dev.mbox"
|
||||
if err := os.WriteFile(path, []byte(cliMbox), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := e.ok(t, "archive", "import", "dev@example.org", path)
|
||||
if !strings.Contains(out, "Imported 2 messages into dev@example.org") || strings.Contains(out, "failed") || strings.Contains(out, "already") {
|
||||
t.Fatalf("import: %q", out)
|
||||
}
|
||||
l, _ := e.s.FindList("dev@example.org")
|
||||
if _, total, _ := e.s.ListMessages(l.ID, 10, 0); total != 2 {
|
||||
t.Fatalf("stored %d", total)
|
||||
}
|
||||
|
||||
// Again, through stdin: nothing new, and it says so.
|
||||
code, out, errs := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}, cliMbox, "archive", "import", "1", "-")
|
||||
if code != 0 || !strings.Contains(out, "Imported 0 messages") || !strings.Contains(out, "2 already present") {
|
||||
t.Fatalf("second import: %d %q %q", code, out, errs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveImportReportsFailures(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
e.ok(t, "list", "add", "dev@example.org")
|
||||
bad := cliMbox + "\nFrom x@y.z Tue Jan 2 15:04:05 2024\n\nno headers\n"
|
||||
code, out, errs := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}, bad, "archive", "import", "dev@example.org", "-")
|
||||
if code != 1 {
|
||||
t.Fatalf("a failed message must give a non-zero exit status, got %d", code)
|
||||
}
|
||||
if !strings.Contains(out, "Imported 2 messages") || !strings.Contains(out, "1 failed") {
|
||||
t.Fatalf("summary: %q", out)
|
||||
}
|
||||
if !strings.Contains(errs, "message 3") || !strings.Contains(errs, "no headers") {
|
||||
t.Fatalf("the failure is reported: %q", errs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveImportErrorsAndJSON(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
e.ok(t, "list", "add", "dev@example.org")
|
||||
e.fails(t, 1, "no such file", "archive", "import", "dev@example.org", "/nonexistent/x.mbox")
|
||||
e.fails(t, 1, "no such list", "archive", "import", "ghost@example.org", "/dev/null")
|
||||
e.fails(t, 2, "usage: parsimony archive import", "archive", "import", "dev@example.org")
|
||||
_, out, _ := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}, cliMbox, "--json", "archive", "import", "dev@example.org", "-")
|
||||
var sum map[string]any
|
||||
if err := json.Unmarshal([]byte(out), &sum); err != nil || sum["imported"].(float64) != 2 {
|
||||
t.Fatalf("json: %v %q", err, out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLongRunningUploadsAreNotCutByTheShortRequestTimeout(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
e.ok(t, "list", "add", "dev@example.org")
|
||||
var cl *client
|
||||
c := &cmd{getenv: func(k string) string {
|
||||
return map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}[k]
|
||||
}}
|
||||
cl, err := c.client()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cl.http.Timeout == 0 {
|
||||
t.Fatal("ordinary requests need a timeout")
|
||||
}
|
||||
if cl.upload == nil || cl.upload.Timeout != 0 {
|
||||
t.Fatal("uploads of big archives must not have an overall timeout")
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveImportSaysHowManyMoreFailuresThereAre(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
e.ok(t, "list", "add", "dev@example.org")
|
||||
var b strings.Builder
|
||||
for i := 0; i < 25; i++ {
|
||||
b.WriteString("From x@y.z Tue Jan 2 15:04:05 2024\n\nno headers\n\n")
|
||||
}
|
||||
code, out, errs := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}, b.String(), "archive", "import", "dev@example.org", "-")
|
||||
if code != 1 || !strings.Contains(out, "25 failed") {
|
||||
t.Fatalf("%d %q", code, out)
|
||||
}
|
||||
if !strings.Contains(errs, "... and 5 more") {
|
||||
t.Fatalf("only 20 are listed, the rest is counted: %q", errs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveImportSendsTheFileAsAnMbox(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
e.ok(t, "list", "add", "dev@example.org")
|
||||
var contentType, body string
|
||||
spy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if strings.HasSuffix(r.URL.Path, "/archive/import") {
|
||||
contentType = r.Header.Get("Content-Type")
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
body = string(b)
|
||||
r.Body = io.NopCloser(strings.NewReader(body))
|
||||
}
|
||||
e.srv.Config.Handler.ServeHTTP(w, r)
|
||||
}))
|
||||
defer spy.Close()
|
||||
vars := map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": spy.URL}
|
||||
if code, _, errs := e.runEnv(t, vars, cliMbox, "archive", "import", "dev@example.org", "-"); code != 0 {
|
||||
t.Fatalf("%d %s", code, errs)
|
||||
}
|
||||
if contentType != "application/mbox" || body != cliMbox {
|
||||
t.Fatalf("content type %q, body unchanged: %v", contentType, body == cliMbox)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -546,6 +546,65 @@ func (c *cmd) archiveRm(args []string) int {
|
|||
}, "Deleted archived message %d")
|
||||
}
|
||||
|
||||
func (c *cmd) archiveImport(args []string) int {
|
||||
pos, err := parseArgs(newFlagSet("archive import"), args)
|
||||
if err != nil {
|
||||
return c.usageError("%v", err)
|
||||
}
|
||||
if len(pos) != 2 {
|
||||
return c.usageError("usage: parsimony archive import <list> <file|->")
|
||||
}
|
||||
l, err := c.resolveList(pos[0])
|
||||
if err != nil {
|
||||
return c.fail(err)
|
||||
}
|
||||
var in io.Reader = c.io.In
|
||||
if pos[1] != "-" {
|
||||
f, err := os.Open(pos[1])
|
||||
if err != nil {
|
||||
return c.fail(err)
|
||||
}
|
||||
defer f.Close()
|
||||
in = f
|
||||
}
|
||||
cl, _ := c.client()
|
||||
raw, err := cl.uploadTo(fmt.Sprintf("/lists/%d/archive/import", l.ID), in, "application/mbox")
|
||||
if err != nil {
|
||||
return c.fail(err)
|
||||
}
|
||||
var sum struct {
|
||||
Imported int `json:"imported"`
|
||||
Duplicates int `json:"duplicates"`
|
||||
Failed int `json:"failed"`
|
||||
Errors []string `json:"errors"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &sum); err != nil {
|
||||
return c.fail(err)
|
||||
}
|
||||
if c.json {
|
||||
c.printJSON(raw)
|
||||
} else {
|
||||
msg := fmt.Sprintf("Imported %s into %s", plural(sum.Imported, "message", "messages"), l.Address)
|
||||
if sum.Duplicates > 0 {
|
||||
msg += fmt.Sprintf(", %d already present", sum.Duplicates)
|
||||
}
|
||||
if sum.Failed > 0 {
|
||||
msg += fmt.Sprintf(", %d failed", sum.Failed)
|
||||
}
|
||||
fmt.Fprintln(c.io.Out, msg)
|
||||
}
|
||||
for _, e := range sum.Errors {
|
||||
fmt.Fprintln(c.io.Err, e)
|
||||
}
|
||||
if sum.Failed > len(sum.Errors) {
|
||||
fmt.Fprintf(c.io.Err, "... and %d more\n", sum.Failed-len(sum.Errors))
|
||||
}
|
||||
if sum.Failed > 0 {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// --- queue --------------------------------------------------------------------------
|
||||
|
||||
func (c *cmd) queueLs(args []string) int {
|
||||
|
|
|
|||
147
internal/mbox/mbox.go
Normal file
147
internal/mbox/mbox.go
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
// Package mbox reads mbox archives, such as the ones Mailman exports.
|
||||
//
|
||||
// A message starts at a line like "From alice@example.com Tue Jan 2 15:04:05 2024".
|
||||
// A line that merely begins with "From " in prose does not split a message. Lines
|
||||
// escaped as ">From " are restored (mboxrd); the blank line before a separator
|
||||
// belongs to the separator, not to the message.
|
||||
package mbox
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrTooLarge is returned with a message that exceeded the size limit and was skipped.
|
||||
var ErrTooLarge = errors.New("message too large")
|
||||
|
||||
var (
|
||||
separator = regexp.MustCompile(`^From \S+\s+(Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2}\s+\d{1,2}:\d{2}`)
|
||||
escaped = regexp.MustCompile(`^>+From `)
|
||||
)
|
||||
|
||||
// Message is one message of the mbox.
|
||||
type Message struct {
|
||||
Index int // 1-based position in the file
|
||||
Envelope string // the "From " line
|
||||
Data []byte // the RFC 5322 message; nil when Err is set
|
||||
Err error
|
||||
}
|
||||
|
||||
// Date returns the date of the "From " line, when it has a parsable one.
|
||||
func (m Message) Date() (time.Time, bool) {
|
||||
f := strings.Fields(m.Envelope)
|
||||
if len(f) < 3 || f[0] != "From" {
|
||||
return time.Time{}, false
|
||||
}
|
||||
rest := strings.Join(f[2:], " ")
|
||||
for _, layout := range []string{
|
||||
"Mon Jan 2 15:04:05 2006",
|
||||
"Mon Jan 2 15:04:05 -0700 2006",
|
||||
"Mon Jan 2 15:04:05 MST 2006",
|
||||
"Mon Jan 2 15:04 2006",
|
||||
} {
|
||||
if t, err := time.Parse(layout, rest); err == nil {
|
||||
return t, true
|
||||
}
|
||||
}
|
||||
return time.Time{}, false
|
||||
}
|
||||
|
||||
// Reader returns the messages of an mbox one at a time.
|
||||
type Reader struct {
|
||||
br *bufio.Reader
|
||||
max int
|
||||
next string // envelope of the message that comes next, once seen
|
||||
index int
|
||||
eof bool
|
||||
}
|
||||
|
||||
// NewReader reads from r. Messages larger than maxSize bytes (0 means no limit)
|
||||
// are skipped and reported with ErrTooLarge.
|
||||
func NewReader(r io.Reader, maxSize int) *Reader {
|
||||
return &Reader{br: bufio.NewReaderSize(r, 64*1024), max: maxSize}
|
||||
}
|
||||
|
||||
// readLine returns the next line including its newline. At the end of the input
|
||||
// it returns the last partial line (if any) with io.EOF on the following call.
|
||||
func (r *Reader) readLine() ([]byte, error) {
|
||||
line, err := r.br.ReadBytes('\n')
|
||||
if err == io.EOF && len(line) > 0 {
|
||||
return line, nil
|
||||
}
|
||||
return line, err
|
||||
}
|
||||
|
||||
// Next returns the next message, or io.EOF when there are no more.
|
||||
func (r *Reader) Next() (Message, error) {
|
||||
if r.eof {
|
||||
return Message{}, io.EOF
|
||||
}
|
||||
// Skip anything before the first separator.
|
||||
for r.next == "" {
|
||||
line, err := r.readLine()
|
||||
if err != nil {
|
||||
r.eof = true
|
||||
if err == io.EOF {
|
||||
return Message{}, io.EOF
|
||||
}
|
||||
return Message{}, err
|
||||
}
|
||||
if s := strings.TrimRight(string(line), "\r\n"); separator.MatchString(s) {
|
||||
r.next = s
|
||||
}
|
||||
}
|
||||
r.index++
|
||||
msg := Message{Index: r.index, Envelope: r.next}
|
||||
r.next = ""
|
||||
|
||||
var buf bytes.Buffer
|
||||
oversized := false
|
||||
for {
|
||||
line, err := r.readLine()
|
||||
if err != nil {
|
||||
r.eof = true
|
||||
if err != io.EOF {
|
||||
return Message{}, err
|
||||
}
|
||||
break
|
||||
}
|
||||
if s := strings.TrimRight(string(line), "\r\n"); separator.MatchString(s) {
|
||||
r.next = s
|
||||
break
|
||||
}
|
||||
if oversized {
|
||||
continue
|
||||
}
|
||||
if escaped.Match(line) {
|
||||
line = line[1:]
|
||||
}
|
||||
buf.Write(line)
|
||||
if r.max > 0 && buf.Len() > r.max {
|
||||
oversized = true
|
||||
buf.Reset()
|
||||
}
|
||||
}
|
||||
if oversized {
|
||||
msg.Err = ErrTooLarge
|
||||
return msg, ErrTooLarge
|
||||
}
|
||||
msg.Data = trimSeparatorLine(buf.Bytes())
|
||||
return msg, nil
|
||||
}
|
||||
|
||||
// trimSeparatorLine drops the one blank line that precedes a separator.
|
||||
func trimSeparatorLine(b []byte) []byte {
|
||||
switch {
|
||||
case bytes.HasSuffix(b, []byte("\r\n\r\n")):
|
||||
return b[:len(b)-2]
|
||||
case bytes.HasSuffix(b, []byte("\n\n")):
|
||||
return b[:len(b)-1]
|
||||
}
|
||||
return b
|
||||
}
|
||||
168
internal/mbox/mbox_test.go
Normal file
168
internal/mbox/mbox_test.go
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
package mbox
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func readAll(t *testing.T, input string, maxSize int) []Message {
|
||||
t.Helper()
|
||||
r := NewReader(strings.NewReader(input), maxSize)
|
||||
var out []Message
|
||||
for {
|
||||
m, err := r.Next()
|
||||
if err == io.EOF {
|
||||
return out
|
||||
}
|
||||
if err != nil && !errors.Is(err, ErrTooLarge) {
|
||||
t.Fatalf("next: %v", err)
|
||||
}
|
||||
m.Err = err
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
|
||||
const two = "From alice@example.com Tue Jan 2 15:04:05 2024\n" +
|
||||
"From: alice@example.com\nSubject: one\n\nfirst body\n\n" +
|
||||
"From bob@example.com Wed Jan 3 09:00:00 2024\n" +
|
||||
"From: bob@example.com\nSubject: two\n\nsecond body\n"
|
||||
|
||||
func TestSplitsMessages(t *testing.T) {
|
||||
ms := readAll(t, two, 0)
|
||||
if len(ms) != 2 {
|
||||
t.Fatalf("got %d messages", len(ms))
|
||||
}
|
||||
if string(ms[0].Data) != "From: alice@example.com\nSubject: one\n\nfirst body\n" {
|
||||
t.Fatalf("first = %q", ms[0].Data)
|
||||
}
|
||||
if string(ms[1].Data) != "From: bob@example.com\nSubject: two\n\nsecond body\n" {
|
||||
t.Fatalf("second = %q", ms[1].Data)
|
||||
}
|
||||
if ms[0].Envelope != "From alice@example.com Tue Jan 2 15:04:05 2024" {
|
||||
t.Fatalf("envelope = %q", ms[0].Envelope)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnvelopeDate(t *testing.T) {
|
||||
ms := readAll(t, two, 0)
|
||||
want := time.Date(2024, 1, 2, 15, 4, 5, 0, time.UTC)
|
||||
if got, ok := ms[0].Date(); !ok || !got.Equal(want) {
|
||||
t.Fatalf("date = %v %v", got, ok)
|
||||
}
|
||||
for _, env := range []string{
|
||||
"From x@y.z Tue Jan 2 15:04:05 +0200 2024",
|
||||
"From x@y.z Tue Jan 2 15:04:05 2024",
|
||||
"From x@y.z Tue Jan 2 15:04 2024",
|
||||
"From MAILER-DAEMON Tue Jan 2 15:04:05 2024",
|
||||
} {
|
||||
if _, ok := (Message{Envelope: env}).Date(); !ok {
|
||||
t.Errorf("%q must parse", env)
|
||||
}
|
||||
}
|
||||
for _, env := range []string{"", "From x@y.z", "From x@y.z not a date"} {
|
||||
if _, ok := (Message{Envelope: env}).Date(); ok {
|
||||
t.Errorf("%q must not parse", env)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProseLineStartingWithFromDoesNotSplit(t *testing.T) {
|
||||
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nFrom now on we meet on Fridays.\nFrom: not a separator either\n"
|
||||
ms := readAll(t, in, 0)
|
||||
if len(ms) != 1 || !strings.Contains(string(ms[0].Data), "From now on we meet on Fridays.") {
|
||||
t.Fatalf("messages: %d %q", len(ms), ms[0].Data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMboxrdUnescaping(t *testing.T) {
|
||||
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n>From the log:\n>>From a quoted line\n> From not escaped\nplain\n"
|
||||
ms := readAll(t, in, 0)
|
||||
body := string(ms[0].Data)
|
||||
if !strings.Contains(body, "\nFrom the log:\n") || !strings.Contains(body, "\n>From a quoted line\n") || !strings.Contains(body, "\n> From not escaped\n") {
|
||||
t.Fatalf("body = %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLeadingJunkAndEmptyInput(t *testing.T) {
|
||||
if got := readAll(t, "", 0); len(got) != 0 {
|
||||
t.Fatalf("empty input: %d", len(got))
|
||||
}
|
||||
if got := readAll(t, "\n\nsome preamble\nFrom a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nbody\n", 0); len(got) != 1 {
|
||||
t.Fatalf("preamble must be ignored: %d", len(got))
|
||||
}
|
||||
if got := readAll(t, "just text, no separator at all\n", 0); len(got) != 0 {
|
||||
t.Fatalf("no messages expected: %d", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCRLFAndMissingFinalNewline(t *testing.T) {
|
||||
in := "From a@x.com Tue Jan 2 15:04:05 2024\r\nSubject: s\r\n\r\nbody without newline"
|
||||
ms := readAll(t, in, 0)
|
||||
if len(ms) != 1 || !strings.HasSuffix(string(ms[0].Data), "body without newline") || !strings.Contains(string(ms[0].Data), "Subject: s\r\n") {
|
||||
t.Fatalf("%q", ms[0].Data)
|
||||
}
|
||||
if string(readAll(t, "From a@x.com Tue Jan 2 15:04:05 2024\r\nSubject: one\r\n\r\nb\r\n\r\nFrom b@x.com Tue Jan 2 15:04:05 2024\r\nSubject: two\r\n\r\nb2\r\n", 0)[0].Data) != "Subject: one\r\n\r\nb\r\n" {
|
||||
t.Fatal("the CRLF blank line before a separator belongs to the separator")
|
||||
}
|
||||
crlf := "From a@x.com Tue Jan 2 15:04:05 2024\r\nSubject: one\r\n\r\nb\r\n\r\nFrom b@x.com Tue Jan 2 15:04:05 2024\r\nSubject: two\r\n\r\nb2\r\n"
|
||||
if got := readAll(t, crlf, 0); len(got) != 2 {
|
||||
t.Fatalf("CRLF separators: %d", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestVeryLongLinesAreFine(t *testing.T) {
|
||||
long := strings.Repeat("x", 300_000)
|
||||
ms := readAll(t, "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n"+long+"\n", 0)
|
||||
if len(ms) != 1 || !strings.Contains(string(ms[0].Data), long) {
|
||||
t.Fatal("a long line must survive")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOversizedMessagesAreReportedAndSkipped(t *testing.T) {
|
||||
big := strings.Repeat("y\n", 5000)
|
||||
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: ok1\n\nsmall\n" +
|
||||
"From b@x.com Tue Jan 2 15:04:05 2024\nSubject: big\n\n" + big +
|
||||
"From c@x.com Tue Jan 2 15:04:05 2024\nSubject: ok2\n\nsmall\n"
|
||||
ms := readAll(t, in, 1000)
|
||||
if len(ms) != 3 {
|
||||
t.Fatalf("got %d", len(ms))
|
||||
}
|
||||
if ms[0].Err != nil || ms[2].Err != nil {
|
||||
t.Fatalf("small messages must pass: %v %v", ms[0].Err, ms[2].Err)
|
||||
}
|
||||
if !errors.Is(ms[1].Err, ErrTooLarge) || ms[1].Data != nil {
|
||||
t.Fatalf("big message: %v %d bytes", ms[1].Err, len(ms[1].Data))
|
||||
}
|
||||
if ms[1].Envelope != "From b@x.com Tue Jan 2 15:04:05 2024" || !strings.Contains(string(ms[2].Data), "ok2") {
|
||||
t.Fatalf("the reader must resynchronise after a big message: %+v", ms[2])
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexIsOneBased(t *testing.T) {
|
||||
ms := readAll(t, two, 0)
|
||||
if ms[0].Index != 1 || ms[1].Index != 2 {
|
||||
t.Fatalf("indexes: %d %d", ms[0].Index, ms[1].Index)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadErrorsArePropagated(t *testing.T) {
|
||||
r := NewReader(io.MultiReader(strings.NewReader("From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nbody\n"), errReader{}), 0)
|
||||
var last error
|
||||
for i := 0; i < 5; i++ {
|
||||
_, err := r.Next()
|
||||
if err != nil {
|
||||
last = err
|
||||
break
|
||||
}
|
||||
}
|
||||
if last == nil || last == io.EOF {
|
||||
t.Fatalf("want the read error, got %v", last)
|
||||
}
|
||||
}
|
||||
|
||||
type errReader struct{}
|
||||
|
||||
func (errReader) Read([]byte) (int, error) { return 0, errors.New("disk on fire") }
|
||||
|
|
@ -2,6 +2,7 @@ package store
|
|||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
|
@ -227,3 +228,207 @@ func TestLongBodyIsStoredAsIs(t *testing.T) {
|
|||
t.Fatalf("body len %d", len(got.Body))
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveBatchInsertsDeduplicatesAndThreads(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
s.ArchiveMessage(l.ID, msg("<old@x>", "Already here", "x"))
|
||||
|
||||
reply := msg("<r@x>", "Re: Topic", "reply")
|
||||
reply.InReplyTo = "<root@x>"
|
||||
res, err := s.ArchiveBatch(l.ID, []Message{
|
||||
msg("<root@x>", "Topic", "root"),
|
||||
reply,
|
||||
msg("<old@x>", "Already here", "x"), // duplicate of an existing message
|
||||
msg("<root@x>", "Topic", "root"), // duplicate inside the batch
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(res) != 4 || !res[0].Created || !res[1].Created || res[2].Created || res[3].Created {
|
||||
t.Fatalf("results: %+v", res)
|
||||
}
|
||||
if res[3].ID != res[0].ID {
|
||||
t.Fatalf("a duplicate reports the id of the stored message: %+v", res)
|
||||
}
|
||||
r, _ := s.GetMessage(res[1].ID)
|
||||
if r.ThreadID != res[0].ID {
|
||||
t.Fatalf("threading inside a batch: thread %d, root %d", r.ThreadID, res[0].ID)
|
||||
}
|
||||
if _, total, _ := s.ListMessages(l.ID, 10, 0); total != 3 {
|
||||
t.Fatalf("total = %d", total)
|
||||
}
|
||||
if got, _, _ := s.SearchMessages(l.ID, "reply", 10, 0); len(got) != 1 {
|
||||
t.Fatal("batch inserts must be searchable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveBatchIsAtomic(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
_, err := s.ArchiveBatch(l.ID, []Message{msg("<a@x>", "A", "a"), {Subject: "no id"}})
|
||||
if err == nil {
|
||||
t.Fatal("a message without an id must fail the batch")
|
||||
}
|
||||
if _, total, _ := s.ListMessages(l.ID, 10, 0); total != 0 {
|
||||
t.Fatalf("nothing may be stored after a failed batch, got %d", total)
|
||||
}
|
||||
if res, err := s.ArchiveBatch(l.ID, nil); err != nil || len(res) != 0 {
|
||||
t.Fatalf("empty batch: %v %v", res, err)
|
||||
}
|
||||
}
|
||||
|
||||
func threadOf(t *testing.T, s *Store, id int64) int64 {
|
||||
t.Helper()
|
||||
m, err := s.GetMessage(id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m.ThreadID
|
||||
}
|
||||
|
||||
func TestRethreadFixesMessagesImportedOutOfOrder(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
// Children first, root last: at insertion time none of the parents exists.
|
||||
c2 := msg("<c2@x>", "Re: Re: T", "c2")
|
||||
c2.InReplyTo = "<c1@x>"
|
||||
c1 := msg("<c1@x>", "Re: T", "c1")
|
||||
c1.InReplyTo = "<root@x>"
|
||||
id2, _, _ := s.ArchiveMessage(l.ID, c2)
|
||||
id1, _, _ := s.ArchiveMessage(l.ID, c1)
|
||||
root, _, _ := s.ArchiveMessage(l.ID, msg("<root@x>", "T", "root"))
|
||||
other, _, _ := s.ArchiveMessage(l.ID, msg("<other@x>", "Other", "o"))
|
||||
if threadOf(t, s, id1) == root {
|
||||
t.Fatal("precondition: the thread is broken before rethreading")
|
||||
}
|
||||
|
||||
changed, err := s.Rethread(l.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if changed != 2 {
|
||||
t.Fatalf("changed = %d, want the 2 broken messages", changed)
|
||||
}
|
||||
for _, id := range []int64{id1, id2, root} {
|
||||
if got := threadOf(t, s, id); got != root {
|
||||
t.Fatalf("message %d thread = %d, want %d", id, got, root)
|
||||
}
|
||||
}
|
||||
if threadOf(t, s, other) != other {
|
||||
t.Fatal("an unrelated message keeps its own thread")
|
||||
}
|
||||
if again, _ := s.Rethread(l.ID); again != 0 {
|
||||
t.Fatalf("rethreading twice changes nothing, got %d", again)
|
||||
}
|
||||
if th, _ := s.ThreadMessages(l.ID, root); len(th) != 3 {
|
||||
t.Fatalf("thread has %d messages", len(th))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRethreadSurvivesCyclesAndSelfReplies(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
a := msg("<a@x>", "A", "a")
|
||||
a.InReplyTo = "<b@x>"
|
||||
b := msg("<b@x>", "B", "b")
|
||||
b.InReplyTo = "<a@x>"
|
||||
self := msg("<s@x>", "Self", "s")
|
||||
self.InReplyTo = "<s@x>"
|
||||
ia, _, _ := s.ArchiveMessage(l.ID, a)
|
||||
ib, _, _ := s.ArchiveMessage(l.ID, b)
|
||||
is, _, _ := s.ArchiveMessage(l.ID, self)
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { _, err := s.Rethread(l.ID); done <- err }()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("Rethread looped forever on a cycle")
|
||||
}
|
||||
if threadOf(t, s, ia) != threadOf(t, s, ib) {
|
||||
t.Fatal("the two messages of a cycle share a thread")
|
||||
}
|
||||
if got := threadOf(t, s, ia); got != ia {
|
||||
t.Fatalf("the cycle's thread is its smallest id (%d), got %d", ia, got)
|
||||
}
|
||||
if threadOf(t, s, is) != is {
|
||||
t.Fatal("a message that replies to itself is its own thread")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRethreadOnlyTouchesTheGivenList(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
a, _ := s.CreateList("a", "example.org", "A")
|
||||
b, _ := s.CreateList("b", "example.org", "B")
|
||||
child := msg("<c@x>", "Re", "c")
|
||||
child.InReplyTo = "<p@x>"
|
||||
cb, _, _ := s.ArchiveMessage(b.ID, child)
|
||||
s.ArchiveMessage(b.ID, msg("<p@x>", "P", "p"))
|
||||
// A parent with the same Message-ID in another list must not be used.
|
||||
s.ArchiveMessage(a.ID, msg("<p@x>", "P in a", "p"))
|
||||
|
||||
if n, _ := s.Rethread(a.ID); n != 0 {
|
||||
t.Fatalf("list a has nothing to fix, changed %d", n)
|
||||
}
|
||||
if threadOf(t, s, cb) != cb {
|
||||
t.Fatal("list b must be untouched by rethreading list a")
|
||||
}
|
||||
if n, _ := s.Rethread(b.ID); n != 1 {
|
||||
t.Fatalf("list b: changed %d", n)
|
||||
}
|
||||
if threadOf(t, s, cb) == cb {
|
||||
t.Fatal("list b's child must now follow its parent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRethreadCycleRootIsTheSmallestIdEvenWhenEnteredFromElsewhere(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
// ids: x=1 replies to c; b=2 replies to c; c=3 replies to b. The cycle is {b, c}, entered from x at c,
|
||||
// which is not its smallest id.
|
||||
x := msg("<x@x>", "X", "x")
|
||||
x.InReplyTo = "<c@x>"
|
||||
b := msg("<b@x>", "B", "b")
|
||||
b.InReplyTo = "<c@x>"
|
||||
c := msg("<c@x>", "C", "c")
|
||||
c.InReplyTo = "<b@x>"
|
||||
ix, _, _ := s.ArchiveMessage(l.ID, x)
|
||||
ib, _, _ := s.ArchiveMessage(l.ID, b)
|
||||
ic, _, _ := s.ArchiveMessage(l.ID, c)
|
||||
if _, err := s.Rethread(l.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := min(ib, ic)
|
||||
for _, id := range []int64{ix, ib, ic} {
|
||||
if got := threadOf(t, s, id); got != want {
|
||||
t.Fatalf("message %d: thread %d, want %d (the smallest id of the cycle)", id, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRethreadIsLinearOnLongChains(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
const n = 20000
|
||||
ms := make([]Message, n)
|
||||
for i := range ms {
|
||||
ms[i] = msg(fmt.Sprintf("<m%d@x>", i), "S", "b")
|
||||
if i > 0 {
|
||||
ms[i].InReplyTo = fmt.Sprintf("<m%d@x>", i-1)
|
||||
}
|
||||
}
|
||||
if _, err := s.ArchiveBatch(l.ID, ms); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
done := make(chan struct{})
|
||||
go func() { s.Rethread(l.ID); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Rethread is not linear: a 20000-message chain took more than 5 seconds")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -590,14 +590,52 @@ func scanMessage(row interface{ Scan(...any) error }, withBody bool) (Message, e
|
|||
return m, err
|
||||
}
|
||||
|
||||
// querier is what archiveOne needs from a *sql.DB or a *sql.Tx.
|
||||
type querier interface {
|
||||
Exec(query string, args ...any) (sql.Result, error)
|
||||
QueryRow(query string, args ...any) *sql.Row
|
||||
}
|
||||
|
||||
// ArchiveResult is the outcome for one message of ArchiveBatch.
|
||||
type ArchiveResult struct {
|
||||
ID int64
|
||||
Created bool
|
||||
}
|
||||
|
||||
// ArchiveMessage stores a post. A message already archived in the same list
|
||||
// (same Message-ID) is not stored again: created is false and the existing id
|
||||
// is returned, so MTA retries are harmless.
|
||||
func (s *Store) ArchiveMessage(listID int64, m Message) (id int64, created bool, err error) {
|
||||
return archiveOne(s.db, listID, m)
|
||||
}
|
||||
|
||||
// ArchiveBatch stores many messages in one transaction: all or none. It is much
|
||||
// faster than ArchiveMessage in a loop, because SQLite syncs once per commit.
|
||||
func (s *Store) ArchiveBatch(listID int64, ms []Message) ([]ArchiveResult, error) {
|
||||
if len(ms) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
out := make([]ArchiveResult, 0, len(ms))
|
||||
for _, m := range ms {
|
||||
id, created, err := archiveOne(tx, listID, m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, ArchiveResult{ID: id, Created: created})
|
||||
}
|
||||
return out, tx.Commit()
|
||||
}
|
||||
|
||||
func archiveOne(q querier, listID int64, m Message) (id int64, created bool, err error) {
|
||||
if m.MessageID == "" {
|
||||
return 0, false, errors.New("message id is required")
|
||||
}
|
||||
err = s.db.QueryRow(`SELECT id FROM archive WHERE list_id=? AND message_id=?`, listID, m.MessageID).Scan(&id)
|
||||
err = q.QueryRow(`SELECT id FROM archive WHERE list_id=? AND message_id=?`, listID, m.MessageID).Scan(&id)
|
||||
if err == nil {
|
||||
return id, false, nil
|
||||
}
|
||||
|
|
@ -606,12 +644,12 @@ func (s *Store) ArchiveMessage(listID int64, m Message) (id int64, created bool,
|
|||
}
|
||||
var thread int64
|
||||
if m.InReplyTo != "" {
|
||||
e := s.db.QueryRow(`SELECT thread_id FROM archive WHERE list_id=? AND message_id=?`, listID, m.InReplyTo).Scan(&thread)
|
||||
e := q.QueryRow(`SELECT thread_id FROM archive WHERE list_id=? AND message_id=?`, listID, m.InReplyTo).Scan(&thread)
|
||||
if e != nil && !errors.Is(e, sql.ErrNoRows) {
|
||||
return 0, false, e
|
||||
}
|
||||
}
|
||||
res, err := s.db.Exec(`INSERT INTO archive(list_id, thread_id, message_id, in_reply_to, subject, sender, date, body, size)
|
||||
res, err := q.Exec(`INSERT INTO archive(list_id, thread_id, message_id, in_reply_to, subject, sender, date, body, size)
|
||||
VALUES(?,?,?,?,?,?,?,?,?)`,
|
||||
listID, thread, m.MessageID, m.InReplyTo, m.Subject, m.Sender, m.Date.Unix(), m.Body, len(m.Body))
|
||||
if err != nil {
|
||||
|
|
@ -619,13 +657,102 @@ func (s *Store) ArchiveMessage(listID int64, m Message) (id int64, created bool,
|
|||
}
|
||||
id, _ = res.LastInsertId()
|
||||
if thread == 0 {
|
||||
if _, err := s.db.Exec(`UPDATE archive SET thread_id=id WHERE id=?`, id); err != nil {
|
||||
if _, err := q.Exec(`UPDATE archive SET thread_id=id WHERE id=?`, id); err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
}
|
||||
return id, true, nil
|
||||
}
|
||||
|
||||
// Rethread recomputes the thread of every message of a list from the In-Reply-To
|
||||
// links, and returns how many messages changed thread. Insertion order does not
|
||||
// matter: a message imported before its parent ends up in the parent's thread.
|
||||
// A cycle of replies becomes one thread whose id is the smallest id in the cycle.
|
||||
func (s *Store) Rethread(listID int64) (int, error) {
|
||||
rows, err := s.db.Query(`SELECT id, message_id, in_reply_to, thread_id FROM archive WHERE list_id=?`, listID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
type row struct {
|
||||
id, thread int64
|
||||
replyTo string
|
||||
}
|
||||
var all []row
|
||||
byMsgID := map[string]int64{}
|
||||
for rows.Next() {
|
||||
var r row
|
||||
var msgID string
|
||||
if err := rows.Scan(&r.id, &msgID, &r.replyTo, &r.thread); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
all = append(all, r)
|
||||
byMsgID[msgID] = r.id
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
parent := map[int64]int64{}
|
||||
for _, r := range all {
|
||||
if p, ok := byMsgID[r.replyTo]; ok && r.replyTo != "" {
|
||||
parent[r.id] = p
|
||||
}
|
||||
}
|
||||
roots := map[int64]int64{}
|
||||
root := func(start int64) int64 {
|
||||
if r, ok := roots[start]; ok {
|
||||
return r
|
||||
}
|
||||
path := []int64{start}
|
||||
seen := map[int64]int{start: 0}
|
||||
cur := start
|
||||
var result int64
|
||||
for {
|
||||
p, ok := parent[cur]
|
||||
if !ok {
|
||||
result = cur
|
||||
break
|
||||
}
|
||||
if r, known := roots[p]; known {
|
||||
result = r
|
||||
break
|
||||
}
|
||||
if at, loop := seen[p]; loop { // a cycle: its smallest id is the root
|
||||
result = path[at]
|
||||
for _, id := range path[at:] {
|
||||
result = min(result, id)
|
||||
}
|
||||
break
|
||||
}
|
||||
seen[p] = len(path)
|
||||
path = append(path, p)
|
||||
cur = p
|
||||
}
|
||||
for _, id := range path {
|
||||
roots[id] = result
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
changed := 0
|
||||
for _, r := range all {
|
||||
if want := root(r.id); want != r.thread {
|
||||
if _, err := tx.Exec(`UPDATE archive SET thread_id=? WHERE id=?`, want, r.id); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
changed++
|
||||
}
|
||||
}
|
||||
return changed, tx.Commit()
|
||||
}
|
||||
|
||||
// GetMessage returns an archived message with its body.
|
||||
func (s *Store) GetMessage(id int64) (Message, error) {
|
||||
return scanMessage(s.db.QueryRow(`SELECT `+msgCols+`, body FROM archive WHERE id=?`, id), true)
|
||||
|
|
|
|||
Loading…
Reference in a new issue