Merge pull request 'Import archives from mbox files (Mailman migration)' (#23) from feat/archive-import into main

This commit is contained in:
squidd 2026-11-09 19:53:57 +01:00
commit fb313736bb
19 changed files with 1684 additions and 20 deletions

View file

@ -61,6 +61,7 @@ parsimony member import dev@example.org subscribers.txt # "email[,name]" per l
parsimony held ls dev@example.org
parsimony held approve 12
parsimony archive ls dev@example.org --q "deploy"
parsimony archive import dev@example.org dev.mbox # an mbox file or "-" for stdin
parsimony queue ls
parsimony --json list ls # JSON for scripts
parsimony help
@ -71,6 +72,23 @@ Exit status: 0 success, 1 failure (including lines that failed in `member import
`parsimony serve -config FILE` runs the daemon; the old `parsimony -config FILE` form still works.
In the container: `docker exec parsimony /parsimony list ls`.
## Migrating from Mailman
Both Mailman 2 and Mailman 3 (HyperKitty) can export a list's history as an mbox file.
```
parsimony list add dev@example.org --title "Developers"
parsimony member import dev@example.org subscribers.txt # "email[,name]" per line
parsimony archive import dev@example.org dev.mbox
```
The import reads standard mbox (a message starts at a `From addr Tue Jan 2 15:04:05 2024` line, `>From ` escapes are
restored), decodes encoded headers, charsets and multipart bodies, and stores the plain text. It is safe to repeat: a
message already in the archive (same `Message-ID`) is skipped. Threads are rebuilt from `In-Reply-To` whatever the order
of the messages. Messages that cannot be read are listed and counted, and the exit status is 1; the others are kept.
It works whatever the list's archive policy is (the policy only governs live mail). The admin UI has the same import on
the Archive tab. Search is accent-insensitive: `caffe` finds "Caffè".
## Docker
The image runs as an unprivileged user (uid 65532) and works with a read-only root filesystem.

View file

@ -159,4 +159,17 @@ describe('endpoints', () => {
['GET', '/api/v1/queue'],
])
})
it('uploads an mbox as the raw body with its own content type', async () => {
mockFetch(respond(200, { imported: 2, duplicates: 0, failed: 0, errors: [] }))
const file = new File(['From a@x.com Tue Jan 2 15:04:05 2024\nSubject: x\n\nbody\n'], 'dev.mbox')
const sum = await api.archive.import(7, file)
expect(sum).toEqual({ imported: 2, duplicates: 0, failed: 0, errors: [] })
expect(calls[0].url).toBe('/api/v1/lists/7/archive/import')
expect(calls[0].init.method).toBe('POST')
expect(calls[0].init.body).toBe(file) // streamed as is, not JSON-encoded
const headers = calls[0].init.headers as Record<string, string>
expect(headers['Content-Type']).toBe('application/mbox')
expect(headers.Authorization).toBe('Bearer secret-token')
})
})

View file

@ -42,6 +42,13 @@ export interface Message {
body?: string
}
export interface ImportSummary {
imported: number
duplicates: number
failed: number
errors: string[]
}
export interface ArchivePage {
total: number
messages: Message[]
@ -90,14 +97,16 @@ async function request<T>(method: string, path: string, body?: unknown): Promise
const headers: Record<string, string> = {}
const token = session.get()
if (token) headers.Authorization = `Bearer ${token}`
if (body !== undefined) headers['Content-Type'] = 'application/json'
// A file is streamed as is; everything else is JSON.
const raw = body instanceof Blob
if (body !== undefined) headers['Content-Type'] = raw ? 'application/mbox' : 'application/json'
let res: Response
try {
res = await fetch(`/api/v1${path}`, {
method,
headers,
body: body === undefined ? undefined : JSON.stringify(body),
body: body === undefined ? undefined : raw ? body : JSON.stringify(body),
})
} catch {
throw new ApiError(0, 'Cannot reach the server')
@ -159,6 +168,7 @@ export const api = {
const s = qs.toString()
return request<ArchivePage>('GET', `/lists/${listId}/archive${s ? `?${s}` : ''}`)
},
import: (listId: number, file: Blob) => request<ImportSummary>('POST', `/lists/${listId}/archive/import`, file),
get: (id: number) => request<Message>('GET', `/archive/${id}`),
remove: (id: number) => request<void>('DELETE', `/archive/${id}`),
},

View file

@ -1,5 +1,5 @@
import { screen, waitFor, within } from '@testing-library/react'
import { describe, expect, it } from 'vitest'
import { describe, expect, it, vi } from 'vitest'
import type { Message } from '@/lib/api'
import { aList, installFakeApi, newState } from '@/test/fakeApi'
import { renderApp } from '@/test/render'
@ -171,4 +171,114 @@ describe('archive tab', () => {
expect(state.archive[1]).toHaveLength(2)
})
})
describe('importing an mbox', () => {
const entry = (n: number) =>
`From user${n}@x.com Tue Jan 2 15:04:05 2024\nFrom: User ${n} <user${n}@x.com>\nSubject: Imported ${n}\nMessage-ID: <imp${n}@x>\n\nbody ${n}\n\n`
const mboxFile = (...ns: number[]) => new File([ns.map(entry).join('')], 'dev.mbox', { type: 'application/mbox' })
it('uploads the file, shows what happened and lists the new messages', async () => {
const { user, state } = await setup([])
await screen.findByText(/no messages archived yet/i)
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1, 2, 3))
const status = await screen.findByRole('status')
expect(status).toHaveTextContent('Imported 3 messages')
expect(state.importTypes).toEqual(['application/mbox'])
expect(state.archive[1]).toHaveLength(3)
expect(await screen.findByRole('button', { name: 'Imported 3' })).toBeInTheDocument()
expect(screen.getByRole('heading', { name: 'Archive (3)', level: 2 })).toBeInTheDocument()
})
it('says when messages were already there', async () => {
const { user } = await setup([])
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1, 2))
await screen.findByText(/Imported 2 messages/)
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1, 2, 3))
const status = await screen.findByText(/Imported 1 message\b/)
expect(status).toHaveTextContent('2 already present')
})
it('lists the messages that could not be read', async () => {
const { user } = await setup([])
const file = new File([entry(1) + 'From x@y.z Tue Jan 2 15:04:05 2024\n\nno headers here\n'], 'bad.mbox')
await user.upload(screen.getByLabelText(/mbox file/i), file)
const status = await screen.findByRole('status')
expect(status).toHaveTextContent('1 failed')
expect(within(status).getByText(/message 2: parse message: no headers/)).toBeInTheDocument()
})
it('shows a server error', async () => {
const { user, state } = await setup([])
state.fail['POST /lists/1/archive/import'] = { status: 500, error: 'disk full' }
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1))
expect(await screen.findByRole('alert')).toHaveTextContent('disk full')
expect(screen.queryByRole('status')).not.toBeInTheDocument()
})
it('shows that it is working, and cannot be started twice', async () => {
let release!: () => void
const { user, state } = await setup([])
state.hold = new Promise<void>((resolve) => (release = resolve))
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1))
const busy = await screen.findByRole('button', { name: /importing/i })
expect(busy).toBeDisabled()
release()
expect(await screen.findByRole('status')).toHaveTextContent('Imported 1 message')
expect(screen.getByRole('button', { name: /^import mbox$/i })).toBeEnabled()
})
it('opens the file chooser from the button and accepts mbox-like files', async () => {
await setup([])
const input = screen.getByLabelText(/mbox file/i) as HTMLInputElement
expect(input.type).toBe('file')
expect(input.accept).toMatch(/\.mbox/)
expect(screen.getByRole('button', { name: /^import mbox$/i })).toBeInTheDocument()
})
it('can import the same file again after a first attempt (the input is reset)', async () => {
const { user, state } = await setup([])
const file = mboxFile(1)
await user.upload(screen.getByLabelText(/mbox file/i), file)
await screen.findByText(/Imported 1 message/)
await user.upload(screen.getByLabelText(/mbox file/i), file)
await waitFor(() => expect(state.importTypes).toHaveLength(2))
})
it('forgets the previous result when a new import starts', async () => {
const { user, state } = await setup([])
state.fail['POST /lists/1/archive/import'] = { status: 500, error: 'disk full' }
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1))
expect(await screen.findByRole('alert')).toHaveTextContent('disk full')
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1))
expect(await screen.findByRole('status')).toHaveTextContent('Imported 1 message')
expect(screen.queryByRole('alert')).not.toBeInTheDocument() // the old error is gone
state.fail['POST /lists/1/archive/import'] = { status: 500, error: 'again' }
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(2))
expect(await screen.findByRole('alert')).toHaveTextContent('again')
expect(screen.queryByRole('status')).not.toBeInTheDocument() // the old summary is gone
})
it('says how many more failures there are than the server listed', async () => {
const { user } = await setup([])
const real = fetch
vi.stubGlobal('fetch', vi.fn((url: string, init?: RequestInit) => {
if (url.endsWith('/archive/import')) {
const errors = Array.from({ length: 20 }, (_, i) => `message ${i + 1}: parse message: no headers`)
return Promise.resolve(new Response(JSON.stringify({ imported: 0, duplicates: 0, failed: 25, errors }), {
status: 200,
headers: { 'Content-Type': 'application/json' },
}))
}
return real(url, init)
}))
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1))
const status = await screen.findByRole('status')
expect(status).toHaveTextContent('25 failed')
expect(within(status).getByText(/and 5 more/)).toBeInTheDocument()
expect(within(status).getAllByRole('listitem')).toHaveLength(21)
})
})
})

View file

@ -1,22 +1,48 @@
import { keepPreviousData, useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
import { useState, type FormEvent } from 'react'
import { useRef, useState, type ChangeEvent, type FormEvent } from 'react'
import { useOutletContext } from 'react-router'
import { toast } from 'sonner'
import { Button } from '@/components/ui/button'
import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from '@/components/ui/dialog'
import { Input } from '@/components/ui/input'
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table'
import { api, type List } from '@/lib/api'
import { api, type ImportSummary, type List } from '@/lib/api'
import { formatBytes, formatDate } from '@/lib/format'
const PAGE_SIZE = 25
function plural(n: number, one: string, many: string) {
return `${n} ${n === 1 ? one : many}`
}
export function ArchiveTab() {
const list = useOutletContext<List>()
const [input, setInput] = useState('')
const [q, setQ] = useState('')
const [page, setPage] = useState(0)
const [openId, setOpenId] = useState<number | null>(null)
const queryClient = useQueryClient()
const fileInput = useRef<HTMLInputElement>(null)
const [imported, setImported] = useState<ImportSummary | null>(null)
const [importError, setImportError] = useState<string | null>(null)
const importMbox = useMutation({
mutationFn: (file: File) => api.archive.import(list.id, file),
onSuccess: async (summary) => {
setImported(summary)
await queryClient.invalidateQueries({ queryKey: ['archive', list.id] })
},
onError: (e: Error) => setImportError(e.message),
})
function chooseFile(e: ChangeEvent<HTMLInputElement>) {
const file = e.target.files?.[0]
e.target.value = '' // lets the same file be chosen again
if (!file) return
setImported(null)
setImportError(null)
importMbox.mutate(file)
}
const archive = useQuery({
queryKey: ['archive', list.id, q, page],
@ -43,7 +69,42 @@ export function ArchiveTab() {
return (
<div className="space-y-4">
<h2 className="text-lg font-semibold">Archive{archive.data && !q ? ` (${total})` : ''}</h2>
<div className="flex items-center justify-between">
<h2 className="text-lg font-semibold">Archive{archive.data && !q ? ` (${total})` : ''}</h2>
<input
ref={fileInput}
type="file"
accept=".mbox,.mbx,.txt,application/mbox,text/plain"
aria-label="Mbox file"
className="hidden"
onChange={chooseFile}
/>
<Button variant="outline" disabled={importMbox.isPending} onClick={() => fileInput.current?.click()}>
{importMbox.isPending ? 'Importing…' : 'Import mbox'}
</Button>
</div>
{importError && (
<p role="alert" className="text-sm text-destructive">
{importError}
</p>
)}
{imported && (
<div role="status" className="space-y-1 rounded-md border p-3 text-sm">
<p>
Imported {plural(imported.imported, 'message', 'messages')}
{imported.duplicates > 0 && `, ${imported.duplicates} already present`}
{imported.failed > 0 && `, ${imported.failed} failed`}
</p>
{imported.errors.length > 0 && (
<ul className="list-disc pl-5 text-destructive">
{imported.errors.map((e) => (
<li key={e}>{e}</li>
))}
{imported.failed > imported.errors.length && <li>… and {imported.failed - imported.errors.length} more</li>}
</ul>
)}
</div>
)}
{list.archive_policy === 'off' && (
<p className="rounded-md border p-3 text-sm text-muted-foreground">
Archiving is off for this list: new messages are not stored. Set Archive to private or public in Settings.

View file

@ -11,6 +11,10 @@ export interface FakeState {
queue: QueueItem[]
/** Like the server, the queue listing is capped. */
queueLimit: number
/** Content types seen on archive imports. */
importTypes: string[]
/** When set, archive imports wait for it (to look at the busy state). */
hold?: Promise<void>
nextId: number
/** Every request seen, as "METHOD /path". */
calls: string[]
@ -19,7 +23,7 @@ export interface FakeState {
}
export function newState(overrides: Partial<FakeState> = {}): FakeState {
return { token: 'good', lists: [], members: {}, held: {}, archive: {}, queue: [], queueLimit: 100, nextId: 100, calls: [], fail: {}, ...overrides }
return { token: 'good', lists: [], members: {}, held: {}, archive: {}, queue: [], queueLimit: 100, importTypes: [], nextId: 100, calls: [], fail: {}, ...overrides }
}
export function aList(over: Partial<List> = {}): List {
@ -62,7 +66,8 @@ export function installFakeApi(state: FakeState) {
delete state.fail[key]
return err(f.status, f.error)
}
const body = init.body ? JSON.parse(init.body as string) : {}
// Files (archive imports) are not JSON.
const body = typeof init.body === 'string' && init.body ? JSON.parse(init.body) : {}
let m: RegExpMatchArray | null
if (path === '/queue' && method === 'GET') {
@ -132,6 +137,34 @@ export function installFakeApi(state: FakeState) {
}
return err(404, 'not found')
}
if ((m = path.match(/^\/lists\/(\d+)\/archive\/import$/)) && method === 'POST') {
const id = Number(m[1])
if (!state.lists.some((l) => l.id === id)) return err(404, 'not found')
if (state.hold) await state.hold
state.importTypes.push((init.headers as Record<string, string>)['Content-Type'] ?? '')
const text = typeof init.body === 'string' ? init.body : await (init.body as Blob).text()
const messages = text.split(/^From \S+ .*$/m).slice(1)
const archive = (state.archive[id] ??= [])
let imported = 0
let duplicates = 0
const errors: string[] = []
messages.forEach((raw, i) => {
const subject = /^Subject: (.*)$/m.exec(raw)?.[1]
const messageId = /^Message-ID: (.*)$/m.exec(raw)?.[1]
if (!subject || !messageId) {
errors.push(`message ${i + 1}: parse message: no headers`)
return
}
if (archive.some((a) => a.message_id === messageId)) {
duplicates++
return
}
const nid = state.nextId++
archive.push({ id: nid, thread_id: nid, message_id: messageId, subject, sender: 'Imported', date: '2024-01-01T00:00:00Z', size: raw.length })
imported++
})
return json(200, { imported, duplicates, failed: errors.length, errors })
}
if ((m = path.match(/^\/lists\/(\d+)\/archive$/)) && method === 'GET') {
const id = Number(m[1])
const q = (url.searchParams.get('q') ?? '').toLowerCase()

View file

@ -275,3 +275,32 @@ paths:
responses:
'204': {description: Discarded}
'404': {$ref: '#/components/responses/Error'}
/lists/{id}/archive/import:
parameters:
- {name: id, in: path, required: true, schema: {type: integer}}
post:
summary: Import an mbox file into the archive
description: >
The request body is the mbox file itself (any content type, `application/mbox` by convention), streamed.
Idempotent: a message already archived (same Message-ID) is counted as a duplicate. Threads are rebuilt at the end.
Works whatever the list's archive policy. Messages that cannot be read are reported in the summary and do not
fail the request; a read error while receiving the body does (400), keeping the messages read so far.
requestBody:
required: true
content:
application/mbox:
schema: {type: string, format: binary}
responses:
'200':
description: Summary
content:
application/json:
schema:
type: object
properties:
imported: {type: integer}
duplicates: {type: integer}
failed: {type: integer}
errors: {type: array, items: {type: string}, description: The first 20 failures}
'400': {$ref: '#/components/responses/Error'}
'404': {$ref: '#/components/responses/Error'}

View file

@ -544,6 +544,70 @@ func TestWholePath(t *testing.T) {
}
})
t.Run("a Mailman-style mbox can be imported and browsed", func(t *testing.T) {
mbox := "From MAILER-DAEMON Mon Jan 1 10:00:00 2024\n" +
"From: =?UTF-8?Q?Jos=C3=A9_Garc=C3=ADa?= <jose@old-list.example>\nSubject: =?UTF-8?Q?Caff=C3=A8_a_las_nueve?=\nMessage-ID: <old-1@old-list.example>\nDate: Mon, 01 Jan 2024 10:00:00 +0000\nMIME-Version: 1.0\nContent-Type: multipart/alternative; boundary=BB\n\n" +
"--BB\nContent-Type: text/plain; charset=utf-8\nContent-Transfer-Encoding: quoted-printable\n\nCiao a tutti, scrivetemi a jose@old-list.example\n>From the old archive: caff=C3=A8!\n--BB\nContent-Type: text/html\n\n<p>html copy</p>\n--BB--\n\n" +
"From carol@members.test Tue Jan 2 09:00:00 2024\n" +
"From: Carol <carol@members.test>\nSubject: Re: Caff\xc3\xa8\nMessage-ID: <old-0@old-list.example>\nIn-Reply-To: <old-1@old-list.example>\nDate: Tue, 02 Jan 2024 09:00:00 +0000\n\nsee you there\n"
imp := exec.Command("docker", "exec", "-i", parsimonyC, "/parsimony", "archive", "import", list, "-")
imp.Stdin = strings.NewReader(mbox)
out, err := imp.CombinedOutput()
if err != nil || !strings.Contains(string(out), "Imported 2 messages") {
t.Fatalf("archive import: %v\n%s", err, out)
}
// Importing again adds nothing.
again := exec.Command("docker", "exec", "-i", parsimonyC, "/parsimony", "archive", "import", list, "-")
again.Stdin = strings.NewReader(mbox)
out, err = again.CombinedOutput()
if err != nil || !strings.Contains(string(out), "Imported 0 messages") || !strings.Contains(string(out), "2 already present") {
t.Fatalf("second import: %v\n%s", err, out)
}
var page struct {
Messages []struct {
ID int64 `json:"id"`
ThreadID int64 `json:"thread_id"`
Subject string `json:"subject"`
Sender string `json:"sender"`
} `json:"messages"`
}
mustCall(t, 200, "GET", "/api/v1/lists/1/archive?q=nueve", "", &page)
if len(page.Messages) != 1 || page.Messages[0].Subject != "Caffè a las nueve" || page.Messages[0].Sender != "José García" {
t.Fatalf("encoded headers must be decoded: %+v", page.Messages)
}
root := page.Messages[0]
var msg struct {
Body string `json:"body"`
}
mustCall(t, 200, "GET", fmt.Sprintf("/api/v1/archive/%d", root.ID), "", &msg)
if !strings.Contains(msg.Body, "caffè!") || !strings.Contains(msg.Body, "\nFrom the old archive") || strings.Contains(msg.Body, "html copy") {
t.Fatalf("body: %q", msg.Body)
}
var all struct {
Messages []struct {
ThreadID int64 `json:"thread_id"`
Subject string `json:"subject"`
} `json:"messages"`
}
mustCall(t, 200, "GET", "/api/v1/lists/1/archive?q=caffe", "", &all)
if len(all.Messages) != 2 || all.Messages[0].ThreadID != all.Messages[1].ThreadID {
t.Fatalf("the reply must be in its parent's thread: %+v", all.Messages)
}
// The list's archive is public: imported messages show up on the web, with addresses hidden.
resp, err := http.Get(fmt.Sprintf("%s/archive/%s/%d", webBase, list, root.ID))
if err != nil {
t.Fatal(err)
}
b, _ := io.ReadAll(resp.Body)
resp.Body.Close()
if resp.StatusCode != 200 || !strings.Contains(string(b), "José García") || !strings.Contains(string(b), "jose@…") ||
strings.Contains(string(b), "jose@old-list.example") {
t.Fatalf("public page: %d\n%.700s", resp.StatusCode, b)
}
})
t.Run("the queue ends empty", func(t *testing.T) {
waitQueueEmpty(t)
})

View file

@ -7,6 +7,7 @@ import (
"crypto/subtle"
"encoding/json"
"errors"
"fmt"
"mime"
"net/http"
"net/mail"
@ -14,6 +15,7 @@ import (
"strings"
"time"
"codeberg.org/squidd/parsimony/internal/archive"
"codeberg.org/squidd/parsimony/internal/store"
)
@ -46,6 +48,7 @@ func New(s *store.Store, token string, rel Releaser) http.Handler {
mux.Handle("POST /api/v1/lists/{id}/members", srv.auth(srv.addMember))
mux.Handle("DELETE /api/v1/lists/{id}/members/{address}", srv.auth(srv.removeMember))
mux.Handle("GET /api/v1/lists/{id}/archive", srv.auth(srv.listArchive))
mux.Handle("POST /api/v1/lists/{id}/archive/import", srv.auth(srv.importArchive))
mux.Handle("GET /api/v1/archive/{id}", srv.auth(srv.getArchived))
mux.Handle("DELETE /api/v1/archive/{id}", srv.auth(srv.deleteArchived))
mux.Handle("GET /api/v1/queue", srv.auth(srv.getQueue))
@ -543,3 +546,22 @@ func (s *server) discardQueued(w http.ResponseWriter, r *http.Request) {
}
w.WriteHeader(http.StatusNoContent)
}
// importArchive reads an mbox from the request body, streaming, and archives it.
// Failed messages are reported in the summary; the request itself succeeds.
func (s *server) importArchive(w http.ResponseWriter, r *http.Request) {
id, ok := listID(w, r)
if !ok {
return
}
if _, err := s.store.GetList(id); err != nil {
storeError(w, err)
return
}
sum, err := archive.ImportMbox(s.store, id, r.Body, time.Now())
if err != nil {
writeError(w, http.StatusBadRequest, fmt.Sprintf("import stopped after %d message(s): %v", sum.Imported, err))
return
}
writeJSON(w, http.StatusOK, sum)
}

View file

@ -3,6 +3,7 @@ package api
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
@ -522,3 +523,109 @@ func TestQueueRequiresAuth(t *testing.T) {
expect(t, do(t, h, "GET", "/api/v1/queue", "", false), 401)
expect(t, do(t, h, "DELETE", "/api/v1/queue/1", "", false), 401)
}
const importMbox = "From alice@example.com Tue Jan 2 15:04:05 2024\nFrom: Alice <alice@example.com>\nSubject: Kickoff\nMessage-ID: <1@x>\nDate: Mon, 01 Jan 2024 10:00:00 +0000\n\nlet's start\n\n" +
"From bob@example.com Tue Jan 2 15:04:05 2024\nFrom: Bob <bob@example.com>\nSubject: Re: Kickoff\nMessage-ID: <2@x>\nIn-Reply-To: <1@x>\nDate: Mon, 01 Jan 2024 11:00:00 +0000\n\nagreed\n"
type importJSON struct {
Imported int `json:"imported"`
Duplicates int `json:"duplicates"`
Failed int `json:"failed"`
Errors []string `json:"errors"`
}
func TestArchiveImport(t *testing.T) {
h, s := queueServer(t)
s.CreateList("dev", "example.org", "Dev")
rec := do(t, h, "POST", "/api/v1/lists/1/archive/import", importMbox, true)
expect(t, rec, 200)
var sum importJSON
decode(t, rec, &sum)
if sum.Imported != 2 || sum.Duplicates != 0 || sum.Failed != 0 || sum.Errors == nil || len(sum.Errors) != 0 {
t.Fatalf("summary: %+v (errors must be [] not null)", sum)
}
var page archiveJSON
decode(t, do(t, h, "GET", "/api/v1/lists/1/archive", "", true), &page)
if page.Total != 2 || page.Messages[0].Subject != "Re: Kickoff" {
t.Fatalf("archive: %+v", page)
}
if page.Messages[0].ThreadID != page.Messages[1].ThreadID {
t.Fatal("imported replies must be threaded")
}
rec = do(t, h, "POST", "/api/v1/lists/1/archive/import", importMbox, true)
expect(t, rec, 200)
decode(t, rec, &sum)
if sum.Imported != 0 || sum.Duplicates != 2 {
t.Fatalf("second import: %+v", sum)
}
}
func TestArchiveImportReportsFailuresWithoutFailingTheRequest(t *testing.T) {
h, s := queueServer(t)
s.CreateList("dev", "example.org", "Dev")
body := importMbox + "\nFrom x@y.z Tue Jan 2 15:04:05 2024\n\nno headers\n"
rec := do(t, h, "POST", "/api/v1/lists/1/archive/import", body, true)
expect(t, rec, 200)
var sum importJSON
decode(t, rec, &sum)
if sum.Imported != 2 || sum.Failed != 1 || len(sum.Errors) != 1 || !strings.Contains(sum.Errors[0], "message 3") {
t.Fatalf("summary: %+v", sum)
}
}
func TestArchiveImportEdgeCases(t *testing.T) {
h, s := queueServer(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
s.SetPolicies(l.ID, store.PostMembers, store.JoinOpen, store.ArchiveOff)
expect(t, do(t, h, "POST", "/api/v1/lists/99/archive/import", importMbox, true), 404)
expect(t, do(t, h, "POST", "/api/v1/lists/abc/archive/import", importMbox, true), 404)
expect(t, do(t, h, "POST", "/api/v1/lists/1/archive/import", importMbox, false), 401)
rec := do(t, h, "POST", "/api/v1/lists/1/archive/import", "", true)
expect(t, rec, 200)
var sum importJSON
decode(t, rec, &sum)
if sum.Imported != 0 || sum.Failed != 0 {
t.Fatalf("empty body: %+v", sum)
}
rec = do(t, h, "POST", "/api/v1/lists/1/archive/import", importMbox, true)
expect(t, rec, 200)
decode(t, rec, &sum)
if sum.Imported != 2 {
t.Fatalf("an import works even when the archive policy is off: %+v", sum)
}
}
type failingBody struct {
data string
done bool
}
func (f *failingBody) Read(p []byte) (int, error) {
if !f.done {
f.done = true
return copy(p, f.data), nil
}
return 0, errors.New("connection reset by peer")
}
func TestArchiveImportStoppedByAReadErrorIsAnErrorResponse(t *testing.T) {
h, s := queueServer(t)
s.CreateList("dev", "example.org", "Dev")
req := httptest.NewRequest("POST", "/api/v1/lists/1/archive/import", &failingBody{data: importMbox})
req.Header.Set("Authorization", "Bearer "+token)
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
expect(t, rec, 400)
var e struct{ Error string }
decode(t, rec, &e)
if !strings.Contains(e.Error, "connection reset") || !strings.Contains(e.Error, "1 message") {
t.Fatalf("the error says what happened and how far it got: %q", e.Error)
}
l, _ := s.FindList("dev@example.org")
if _, total, _ := s.ListMessages(l.ID, 10, 0); total != 1 {
t.Fatalf("the first message was complete and is kept, got %d", total)
}
}

106
internal/archive/import.go Normal file
View file

@ -0,0 +1,106 @@
package archive
import (
"errors"
"fmt"
"io"
"time"
"codeberg.org/squidd/parsimony/internal/mbox"
"codeberg.org/squidd/parsimony/internal/store"
)
const (
// MaxImportMessage is the largest message accepted from an mbox, in bytes.
MaxImportMessage = 25 << 20
// MaxReportedErrors caps the error messages kept in a summary; Failed still counts every one.
MaxReportedErrors = 20
importBatch = 200 // messages per transaction
)
// ImportSummary reports what an import did.
type ImportSummary struct {
Imported int `json:"imported"` // new messages stored
Duplicates int `json:"duplicates"` // already in the archive (same Message-ID)
Failed int `json:"failed"` // unreadable or too large
Errors []string `json:"errors"` // the first MaxReportedErrors failures
}
// ImportMbox archives every message of an mbox into a list. It is idempotent
// (a message is identified by its Message-ID) and works whatever the list's
// archive policy is: the policy only governs live mail. Threads are rebuilt at
// the end, so the order of the messages in the file does not matter. now is the
// date of last resort for a message with neither a Date header nor a dated
// "From " line. On a read error the messages read so far are kept and returned
// in the summary together with the error.
func ImportMbox(st *store.Store, listID int64, r io.Reader, now time.Time) (ImportSummary, error) {
sum := ImportSummary{Errors: []string{}}
var batch []store.Message
fail := func(index int, err error) {
sum.Failed++
if len(sum.Errors) < MaxReportedErrors {
sum.Errors = append(sum.Errors, fmt.Sprintf("message %d: %v", index, err))
}
}
flush := func() error {
results, err := st.ArchiveBatch(listID, batch)
if err != nil {
return err
}
for _, res := range results {
if res.Created {
sum.Imported++
} else {
sum.Duplicates++
}
}
batch = batch[:0]
return nil
}
finish := func(readErr error) (ImportSummary, error) {
if err := flush(); err != nil {
return sum, err
}
if sum.Imported > 0 {
if _, err := st.Rethread(listID); err != nil {
return sum, err
}
}
return sum, readErr
}
rd := mbox.NewReader(r, MaxImportMessage)
for {
m, err := rd.Next()
if err == io.EOF {
return finish(nil)
}
if errors.Is(err, mbox.ErrTooLarge) {
fail(m.Index, fmt.Errorf("too large (limit %d MiB)", MaxImportMessage>>20))
continue
}
if err != nil {
return finish(fmt.Errorf("read mbox: %w", err))
}
fallback := now
if d, ok := m.Date(); ok {
fallback = d
}
en, err := Parse(m.Data, fallback)
if err != nil {
fail(m.Index, err)
continue
}
batch = append(batch, store.Message{
MessageID: en.MessageID, InReplyTo: en.InReplyTo, Subject: en.Subject,
Sender: en.Sender, Date: en.Date, Body: en.Body,
})
if len(batch) >= importBatch {
if err := flush(); err != nil {
return sum, err
}
}
}
}

View file

@ -0,0 +1,254 @@
package archive
import (
"errors"
"fmt"
"strings"
"testing"
"time"
"codeberg.org/squidd/parsimony/internal/store"
)
var importNow = time.Date(2031, 5, 6, 7, 8, 9, 0, time.UTC)
func newList(t *testing.T) (*store.Store, store.List) {
t.Helper()
s, err := store.Open(":memory:")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { s.Close() })
l, _ := s.CreateList("dev", "example.org", "Dev")
return s, l
}
func mboxMsg(from, subject, id, replyTo, date, body string) string {
addr := from
if i := strings.Index(from, "<"); i >= 0 {
addr = strings.Trim(from[i:], "<>")
}
h := "From " + addr + " Tue Jan 2 15:04:05 2024\n"
h += "From: " + from + "\nSubject: " + subject + "\n"
if id != "" {
h += "Message-ID: " + id + "\n"
}
if replyTo != "" {
h += "In-Reply-To: " + replyTo + "\n"
}
if date != "" {
h += "Date: " + date + "\n"
}
return h + "\n" + body + "\n\n"
}
func TestImportMbox(t *testing.T) {
s, l := newList(t)
mbox := mboxMsg("Alice <alice@example.com>", "Kickoff", "<1@x>", "", "Mon, 01 Jan 2024 10:00:00 +0000", "let's start") +
mboxMsg("Bob <bob@example.com>", "Re: Kickoff", "<2@x>", "<1@x>", "Mon, 01 Jan 2024 11:00:00 +0000", "agreed") +
mboxMsg("Carol <carol@example.com>", "Lunch", "<3@x>", "", "Tue, 02 Jan 2024 12:00:00 +0000", "pizza?")
sum, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow)
if err != nil {
t.Fatal(err)
}
if sum.Imported != 3 || sum.Duplicates != 0 || sum.Failed != 0 || len(sum.Errors) != 0 {
t.Fatalf("summary: %+v", sum)
}
ms, total, _ := s.ListMessages(l.ID, 10, 0)
if total != 3 || ms[0].Subject != "Lunch" {
t.Fatalf("messages: %+v", ms)
}
var reply store.Message
for _, m := range ms {
if m.Subject == "Re: Kickoff" {
reply, _ = s.GetMessage(m.ID)
}
}
if reply.Sender != "Bob" || reply.Body != "agreed\n" || !reply.Date.Equal(time.Date(2024, 1, 1, 11, 0, 0, 0, time.UTC)) {
t.Fatalf("reply: %+v", reply)
}
root, _ := s.GetMessage(reply.ThreadID)
if root.Subject != "Kickoff" {
t.Fatalf("the reply must be threaded under its parent, got %q", root.Subject)
}
}
func TestImportIsIdempotent(t *testing.T) {
s, l := newList(t)
mbox := mboxMsg("A <a@x.com>", "One", "<1@x>", "", "", "b1") + mboxMsg("B <b@x.com>", "Two", "", "", "", "b2")
if sum, _ := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow); sum.Imported != 2 {
t.Fatalf("first: %+v", sum)
}
sum, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow)
if err != nil || sum.Imported != 0 || sum.Duplicates != 2 || sum.Failed != 0 {
t.Fatalf("second import must add nothing, even for a message without a Message-ID: %+v %v", sum, err)
}
if _, total, _ := s.ListMessages(l.ID, 10, 0); total != 2 {
t.Fatalf("total = %d", total)
}
}
func TestImportRebuildsThreadsWhateverTheOrder(t *testing.T) {
s, l := newList(t)
mbox := mboxMsg("C <c@x.com>", "Re: Re: T", "<c@x>", "<b@x>", "", "c") +
mboxMsg("B <b@x.com>", "Re: T", "<b@x>", "<a@x>", "", "b") +
mboxMsg("A <a@x.com>", "T", "<a@x>", "", "", "a")
if _, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow); err != nil {
t.Fatal(err)
}
ms, _, _ := s.ListMessages(l.ID, 10, 0)
threads := map[int64]int{}
for _, m := range ms {
threads[m.ThreadID]++
}
if len(threads) != 1 {
t.Fatalf("one thread expected, got %v", threads)
}
}
func TestImportReportsBadMessagesAndKeepsGoing(t *testing.T) {
s, l := newList(t)
mbox := mboxMsg("A <a@x.com>", "Good one", "<1@x>", "", "", "ok") +
"From x@y.z Tue Jan 2 15:04:05 2024\n\n\nthis message has no headers at all\n\n" +
mboxMsg("B <b@x.com>", "Good two", "<2@x>", "", "", "ok")
sum, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow)
if err != nil {
t.Fatal(err)
}
if sum.Imported != 2 || sum.Failed != 1 || len(sum.Errors) != 1 || !strings.Contains(sum.Errors[0], "message 2") {
t.Fatalf("summary: %+v", sum)
}
}
func TestImportSkipsOversizedMessages(t *testing.T) {
s, l := newList(t)
big := strings.Repeat("0123456789", MaxImportMessage/10+10)
mbox := mboxMsg("A <a@x.com>", "Small", "<1@x>", "", "", "ok") + mboxMsg("B <b@x.com>", "Huge", "<2@x>", "", "", big) +
mboxMsg("C <c@x.com>", "Small too", "<3@x>", "", "", "ok")
sum, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow)
if err != nil {
t.Fatal(err)
}
if sum.Imported != 2 || sum.Failed != 1 || !strings.Contains(sum.Errors[0], "message 2") || !strings.Contains(sum.Errors[0], "too large") {
t.Fatalf("summary: %+v", sum)
}
}
func TestImportDateFallbacks(t *testing.T) {
s, l := newList(t)
mbox := mboxMsg("A <a@x.com>", "has Date", "<1@x>", "", "Mon, 01 Jan 2024 10:00:00 +0000", "b") +
mboxMsg("B <b@x.com>", "from envelope", "<2@x>", "", "", "b") +
"From c@x.com Fri Feb 23 08:30 2024\nFrom: C <c@x.com>\nSubject: nothing\nMessage-ID: <3@x>\n\nb\n\n"
if _, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow); err != nil {
t.Fatal(err)
}
ms, _, _ := s.ListMessages(l.ID, 10, 0)
dates := map[string]time.Time{}
for _, m := range ms {
dates[m.Subject] = m.Date
}
if !dates["has Date"].Equal(time.Date(2024, 1, 1, 10, 0, 0, 0, time.UTC)) {
t.Errorf("Date header wins: %v", dates["has Date"])
}
if !dates["from envelope"].Equal(time.Date(2024, 1, 2, 15, 4, 5, 0, time.UTC)) {
t.Errorf("the From line is the fallback: %v", dates["from envelope"])
}
if !dates["nothing"].Equal(time.Date(2024, 2, 23, 8, 30, 0, 0, time.UTC)) {
t.Errorf("minute-precision envelope date: %v", dates["nothing"])
}
}
func TestImportDateFallsBackToNowAsALastResort(t *testing.T) {
s, l := newList(t)
// A valid separator whose date has trailing junk, so it cannot be parsed, and no Date header.
mbox := "From a@x.com Tue Jan 2 15:04:05 2024 extra junk after\nFrom: A <a@x.com>\nSubject: undated\nMessage-ID: <1@x>\n\nb\n"
sum, _ := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow)
if sum.Imported != 1 {
t.Fatalf("%+v", sum)
}
ms, _, _ := s.ListMessages(l.ID, 10, 0)
if !ms[0].Date.Equal(importNow) {
t.Fatalf("date = %v, want the import time", ms[0].Date)
}
}
func TestImportCapsTheErrorListButCountsEverything(t *testing.T) {
s, l := newList(t)
var b strings.Builder
for i := 0; i < 30; i++ {
b.WriteString("From x@y.z Tue Jan 2 15:04:05 2024\n\nno headers here\n\n")
}
sum, err := ImportMbox(s, l.ID, strings.NewReader(b.String()), importNow)
if err != nil {
t.Fatal(err)
}
if sum.Failed != 30 || len(sum.Errors) != MaxReportedErrors {
t.Fatalf("failed=%d errors=%d", sum.Failed, len(sum.Errors))
}
}
func TestImportWorksWhateverTheArchivePolicy(t *testing.T) {
s, l := newList(t)
s.SetPolicies(l.ID, store.PostMembers, store.JoinOpen, store.ArchiveOff)
sum, _ := ImportMbox(s, l.ID, strings.NewReader(mboxMsg("A <a@x.com>", "S", "<1@x>", "", "", "b")), importNow)
if sum.Imported != 1 {
t.Fatalf("an explicit import is not governed by the live-mail policy: %+v", sum)
}
}
func TestImportLargeFileCrossesBatchBoundaries(t *testing.T) {
s, l := newList(t)
var b strings.Builder
n := importBatch*2 + 50
for i := 0; i < n; i++ {
parent := ""
if i > 0 && i%7 != 0 {
parent = fmt.Sprintf("<m%d@x>", i-1)
}
b.WriteString(mboxMsg("A <a@x.com>", fmt.Sprintf("Subject %d", i), fmt.Sprintf("<m%d@x>", i), parent, "", "body"))
}
sum, err := ImportMbox(s, l.ID, strings.NewReader(b.String()), importNow)
if err != nil || sum.Imported != n || sum.Failed != 0 {
t.Fatalf("summary: %+v %v", sum, err)
}
if _, total, _ := s.ListMessages(l.ID, 1, 0); total != n {
t.Fatalf("stored %d of %d", total, n)
}
}
func TestImportDoesNotTouchOtherLists(t *testing.T) {
s, l := newList(t)
other, _ := s.CreateList("ops", "example.org", "Ops")
ImportMbox(s, l.ID, strings.NewReader(mboxMsg("A <a@x.com>", "S", "<1@x>", "", "", "b")), importNow)
if _, total, _ := s.ListMessages(other.ID, 10, 0); total != 0 {
t.Fatalf("other list has %d messages", total)
}
}
type brokenReader struct {
data string
done bool
}
func (b *brokenReader) Read(p []byte) (int, error) {
if !b.done {
b.done = true
return copy(p, b.data), nil
}
return 0, errors.New("connection reset")
}
func TestImportReturnsTheProgressMadeBeforeAReadError(t *testing.T) {
s, l := newList(t)
data := mboxMsg("A <a@x.com>", "First", "<1@x>", "", "", "b") + mboxMsg("B <b@x.com>", "Second", "<2@x>", "", "", "b")
sum, err := ImportMbox(s, l.ID, &brokenReader{data: data}, importNow)
if err == nil || !strings.Contains(err.Error(), "connection reset") {
t.Fatalf("want the read error, got %v", err)
}
if sum.Imported != 1 {
t.Fatalf("the messages read before the failure are kept: %+v", sum)
}
if _, total, _ := s.ListMessages(l.ID, 10, 0); total != 1 {
t.Fatalf("stored %d", total)
}
}

View file

@ -45,6 +45,8 @@ Administration (talks to the REST API):
archive ls <list> [--q TEXT] [--limit N] [--offset N]
archive show <id>
archive rm <id> --yes
archive import <list> <file|-> add the messages of an mbox file (Mailman export);
safe to repeat, threads are rebuilt
queue ls outbound queue
queue rm <id> discard one queued delivery
@ -90,7 +92,7 @@ func Run(args []string, io IO, getenv func(string) string) int {
"list": {"ls": c.listLs, "add": c.listAdd, "show": c.listShow, "set": c.listSet, "rm": c.listRm},
"member": {"ls": c.memberLs, "add": c.memberAdd, "rm": c.memberRm, "import": c.memberImport},
"held": {"ls": c.heldLs, "approve": c.heldApprove, "discard": c.heldDiscard},
"archive": {"ls": c.archiveLs, "show": c.archiveShow, "rm": c.archiveRm},
"archive": {"ls": c.archiveLs, "show": c.archiveShow, "rm": c.archiveRm, "import": c.archiveImport},
"queue": {"ls": c.queueLs, "rm": c.queueRm},
}
group, ok := commands[rest[0]]
@ -145,7 +147,11 @@ func (c *cmd) client() (*client, error) {
if _, err := url.Parse(base); err != nil {
return nil, fmt.Errorf("invalid URL %q", base)
}
c.cl = &client{base: strings.TrimRight(base, "/"), token: token, http: &http.Client{Timeout: 30 * time.Second}}
c.cl = &client{
base: strings.TrimRight(base, "/"), token: token,
http: &http.Client{Timeout: 30 * time.Second},
upload: &http.Client{}, // big files: no overall timeout
}
return c.cl, nil
}
@ -176,7 +182,8 @@ func parseArgs(fs *flag.FlagSet, args []string) ([]string, error) {
type client struct {
base, token string
http *http.Client
http *http.Client // ordinary requests, with a timeout
upload *http.Client // streaming uploads, without one
}
type apiError struct {
@ -186,25 +193,35 @@ type apiError struct {
func (e *apiError) Error() string { return e.message }
// do sends a request and returns the raw response body (nil for 204).
// do sends a JSON request and returns the raw response body (nil for 204).
func (cl *client) do(method, path string, body any) (json.RawMessage, error) {
var rd io.Reader
ctype := ""
if body != nil {
b, err := json.Marshal(body)
if err != nil {
return nil, err
}
rd = bytes.NewReader(b)
rd, ctype = bytes.NewReader(b), "application/json"
}
return cl.send(cl.http, method, path, rd, ctype)
}
// uploadTo streams r as the body of a POST, with no overall timeout.
func (cl *client) uploadTo(path string, r io.Reader, contentType string) (json.RawMessage, error) {
return cl.send(cl.upload, "POST", path, r, contentType)
}
func (cl *client) send(hc *http.Client, method, path string, rd io.Reader, contentType string) (json.RawMessage, error) {
req, err := http.NewRequest(method, cl.base+"/api/v1"+path, rd)
if err != nil {
return nil, err
}
req.Header.Set("Authorization", "Bearer "+cl.token)
if body != nil {
req.Header.Set("Content-Type", "application/json")
if contentType != "" {
req.Header.Set("Content-Type", contentType)
}
resp, err := cl.http.Do(req)
resp, err := hc.Do(req)
if err != nil {
return nil, fmt.Errorf("cannot reach %s: %w", cl.base, errors.Unwrap(err))
}

View file

@ -5,6 +5,7 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
@ -438,3 +439,116 @@ func TestQueue(t *testing.T) {
t.Fatalf("pending = %d", st.Pending)
}
}
const cliMbox = "From alice@example.com Tue Jan 2 15:04:05 2024\nFrom: Alice <alice@example.com>\nSubject: Kickoff\nMessage-ID: <1@x>\nDate: Mon, 01 Jan 2024 10:00:00 +0000\n\nlet's start\n\n" +
"From bob@example.com Tue Jan 2 15:04:05 2024\nFrom: Bob <bob@example.com>\nSubject: Re: Kickoff\nMessage-ID: <2@x>\nIn-Reply-To: <1@x>\nDate: Mon, 01 Jan 2024 11:00:00 +0000\n\nagreed\n"
func TestArchiveImportFromFileAndStdin(t *testing.T) {
e := newEnv(t)
e.ok(t, "list", "add", "dev@example.org")
path := t.TempDir() + "/dev.mbox"
if err := os.WriteFile(path, []byte(cliMbox), 0o600); err != nil {
t.Fatal(err)
}
out := e.ok(t, "archive", "import", "dev@example.org", path)
if !strings.Contains(out, "Imported 2 messages into dev@example.org") || strings.Contains(out, "failed") || strings.Contains(out, "already") {
t.Fatalf("import: %q", out)
}
l, _ := e.s.FindList("dev@example.org")
if _, total, _ := e.s.ListMessages(l.ID, 10, 0); total != 2 {
t.Fatalf("stored %d", total)
}
// Again, through stdin: nothing new, and it says so.
code, out, errs := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}, cliMbox, "archive", "import", "1", "-")
if code != 0 || !strings.Contains(out, "Imported 0 messages") || !strings.Contains(out, "2 already present") {
t.Fatalf("second import: %d %q %q", code, out, errs)
}
}
func TestArchiveImportReportsFailures(t *testing.T) {
e := newEnv(t)
e.ok(t, "list", "add", "dev@example.org")
bad := cliMbox + "\nFrom x@y.z Tue Jan 2 15:04:05 2024\n\nno headers\n"
code, out, errs := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}, bad, "archive", "import", "dev@example.org", "-")
if code != 1 {
t.Fatalf("a failed message must give a non-zero exit status, got %d", code)
}
if !strings.Contains(out, "Imported 2 messages") || !strings.Contains(out, "1 failed") {
t.Fatalf("summary: %q", out)
}
if !strings.Contains(errs, "message 3") || !strings.Contains(errs, "no headers") {
t.Fatalf("the failure is reported: %q", errs)
}
}
func TestArchiveImportErrorsAndJSON(t *testing.T) {
e := newEnv(t)
e.ok(t, "list", "add", "dev@example.org")
e.fails(t, 1, "no such file", "archive", "import", "dev@example.org", "/nonexistent/x.mbox")
e.fails(t, 1, "no such list", "archive", "import", "ghost@example.org", "/dev/null")
e.fails(t, 2, "usage: parsimony archive import", "archive", "import", "dev@example.org")
_, out, _ := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}, cliMbox, "--json", "archive", "import", "dev@example.org", "-")
var sum map[string]any
if err := json.Unmarshal([]byte(out), &sum); err != nil || sum["imported"].(float64) != 2 {
t.Fatalf("json: %v %q", err, out)
}
}
func TestLongRunningUploadsAreNotCutByTheShortRequestTimeout(t *testing.T) {
e := newEnv(t)
e.ok(t, "list", "add", "dev@example.org")
var cl *client
c := &cmd{getenv: func(k string) string {
return map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}[k]
}}
cl, err := c.client()
if err != nil {
t.Fatal(err)
}
if cl.http.Timeout == 0 {
t.Fatal("ordinary requests need a timeout")
}
if cl.upload == nil || cl.upload.Timeout != 0 {
t.Fatal("uploads of big archives must not have an overall timeout")
}
}
func TestArchiveImportSaysHowManyMoreFailuresThereAre(t *testing.T) {
e := newEnv(t)
e.ok(t, "list", "add", "dev@example.org")
var b strings.Builder
for i := 0; i < 25; i++ {
b.WriteString("From x@y.z Tue Jan 2 15:04:05 2024\n\nno headers\n\n")
}
code, out, errs := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}, b.String(), "archive", "import", "dev@example.org", "-")
if code != 1 || !strings.Contains(out, "25 failed") {
t.Fatalf("%d %q", code, out)
}
if !strings.Contains(errs, "... and 5 more") {
t.Fatalf("only 20 are listed, the rest is counted: %q", errs)
}
}
func TestArchiveImportSendsTheFileAsAnMbox(t *testing.T) {
e := newEnv(t)
e.ok(t, "list", "add", "dev@example.org")
var contentType, body string
spy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasSuffix(r.URL.Path, "/archive/import") {
contentType = r.Header.Get("Content-Type")
b, _ := io.ReadAll(r.Body)
body = string(b)
r.Body = io.NopCloser(strings.NewReader(body))
}
e.srv.Config.Handler.ServeHTTP(w, r)
}))
defer spy.Close()
vars := map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": spy.URL}
if code, _, errs := e.runEnv(t, vars, cliMbox, "archive", "import", "dev@example.org", "-"); code != 0 {
t.Fatalf("%d %s", code, errs)
}
if contentType != "application/mbox" || body != cliMbox {
t.Fatalf("content type %q, body unchanged: %v", contentType, body == cliMbox)
}
}

View file

@ -546,6 +546,65 @@ func (c *cmd) archiveRm(args []string) int {
}, "Deleted archived message %d")
}
func (c *cmd) archiveImport(args []string) int {
pos, err := parseArgs(newFlagSet("archive import"), args)
if err != nil {
return c.usageError("%v", err)
}
if len(pos) != 2 {
return c.usageError("usage: parsimony archive import <list> <file|->")
}
l, err := c.resolveList(pos[0])
if err != nil {
return c.fail(err)
}
var in io.Reader = c.io.In
if pos[1] != "-" {
f, err := os.Open(pos[1])
if err != nil {
return c.fail(err)
}
defer f.Close()
in = f
}
cl, _ := c.client()
raw, err := cl.uploadTo(fmt.Sprintf("/lists/%d/archive/import", l.ID), in, "application/mbox")
if err != nil {
return c.fail(err)
}
var sum struct {
Imported int `json:"imported"`
Duplicates int `json:"duplicates"`
Failed int `json:"failed"`
Errors []string `json:"errors"`
}
if err := json.Unmarshal(raw, &sum); err != nil {
return c.fail(err)
}
if c.json {
c.printJSON(raw)
} else {
msg := fmt.Sprintf("Imported %s into %s", plural(sum.Imported, "message", "messages"), l.Address)
if sum.Duplicates > 0 {
msg += fmt.Sprintf(", %d already present", sum.Duplicates)
}
if sum.Failed > 0 {
msg += fmt.Sprintf(", %d failed", sum.Failed)
}
fmt.Fprintln(c.io.Out, msg)
}
for _, e := range sum.Errors {
fmt.Fprintln(c.io.Err, e)
}
if sum.Failed > len(sum.Errors) {
fmt.Fprintf(c.io.Err, "... and %d more\n", sum.Failed-len(sum.Errors))
}
if sum.Failed > 0 {
return 1
}
return 0
}
// --- queue --------------------------------------------------------------------------
func (c *cmd) queueLs(args []string) int {

147
internal/mbox/mbox.go Normal file
View file

@ -0,0 +1,147 @@
// Package mbox reads mbox archives, such as the ones Mailman exports.
//
// A message starts at a line like "From alice@example.com Tue Jan 2 15:04:05 2024".
// A line that merely begins with "From " in prose does not split a message. Lines
// escaped as ">From " are restored (mboxrd); the blank line before a separator
// belongs to the separator, not to the message.
package mbox
import (
"bufio"
"bytes"
"errors"
"io"
"regexp"
"strings"
"time"
)
// ErrTooLarge is returned with a message that exceeded the size limit and was skipped.
var ErrTooLarge = errors.New("message too large")
var (
separator = regexp.MustCompile(`^From \S+\s+(Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2}\s+\d{1,2}:\d{2}`)
escaped = regexp.MustCompile(`^>+From `)
)
// Message is one message of the mbox.
type Message struct {
Index int // 1-based position in the file
Envelope string // the "From " line
Data []byte // the RFC 5322 message; nil when Err is set
Err error
}
// Date returns the date of the "From " line, when it has a parsable one.
func (m Message) Date() (time.Time, bool) {
f := strings.Fields(m.Envelope)
if len(f) < 3 || f[0] != "From" {
return time.Time{}, false
}
rest := strings.Join(f[2:], " ")
for _, layout := range []string{
"Mon Jan 2 15:04:05 2006",
"Mon Jan 2 15:04:05 -0700 2006",
"Mon Jan 2 15:04:05 MST 2006",
"Mon Jan 2 15:04 2006",
} {
if t, err := time.Parse(layout, rest); err == nil {
return t, true
}
}
return time.Time{}, false
}
// Reader returns the messages of an mbox one at a time.
type Reader struct {
br *bufio.Reader
max int
next string // envelope of the message that comes next, once seen
index int
eof bool
}
// NewReader reads from r. Messages larger than maxSize bytes (0 means no limit)
// are skipped and reported with ErrTooLarge.
func NewReader(r io.Reader, maxSize int) *Reader {
return &Reader{br: bufio.NewReaderSize(r, 64*1024), max: maxSize}
}
// readLine returns the next line including its newline. At the end of the input
// it returns the last partial line (if any) with io.EOF on the following call.
func (r *Reader) readLine() ([]byte, error) {
line, err := r.br.ReadBytes('\n')
if err == io.EOF && len(line) > 0 {
return line, nil
}
return line, err
}
// Next returns the next message, or io.EOF when there are no more.
func (r *Reader) Next() (Message, error) {
if r.eof {
return Message{}, io.EOF
}
// Skip anything before the first separator.
for r.next == "" {
line, err := r.readLine()
if err != nil {
r.eof = true
if err == io.EOF {
return Message{}, io.EOF
}
return Message{}, err
}
if s := strings.TrimRight(string(line), "\r\n"); separator.MatchString(s) {
r.next = s
}
}
r.index++
msg := Message{Index: r.index, Envelope: r.next}
r.next = ""
var buf bytes.Buffer
oversized := false
for {
line, err := r.readLine()
if err != nil {
r.eof = true
if err != io.EOF {
return Message{}, err
}
break
}
if s := strings.TrimRight(string(line), "\r\n"); separator.MatchString(s) {
r.next = s
break
}
if oversized {
continue
}
if escaped.Match(line) {
line = line[1:]
}
buf.Write(line)
if r.max > 0 && buf.Len() > r.max {
oversized = true
buf.Reset()
}
}
if oversized {
msg.Err = ErrTooLarge
return msg, ErrTooLarge
}
msg.Data = trimSeparatorLine(buf.Bytes())
return msg, nil
}
// trimSeparatorLine drops the one blank line that precedes a separator.
func trimSeparatorLine(b []byte) []byte {
switch {
case bytes.HasSuffix(b, []byte("\r\n\r\n")):
return b[:len(b)-2]
case bytes.HasSuffix(b, []byte("\n\n")):
return b[:len(b)-1]
}
return b
}

168
internal/mbox/mbox_test.go Normal file
View file

@ -0,0 +1,168 @@
package mbox
import (
"errors"
"io"
"strings"
"testing"
"time"
)
func readAll(t *testing.T, input string, maxSize int) []Message {
t.Helper()
r := NewReader(strings.NewReader(input), maxSize)
var out []Message
for {
m, err := r.Next()
if err == io.EOF {
return out
}
if err != nil && !errors.Is(err, ErrTooLarge) {
t.Fatalf("next: %v", err)
}
m.Err = err
out = append(out, m)
}
}
const two = "From alice@example.com Tue Jan 2 15:04:05 2024\n" +
"From: alice@example.com\nSubject: one\n\nfirst body\n\n" +
"From bob@example.com Wed Jan 3 09:00:00 2024\n" +
"From: bob@example.com\nSubject: two\n\nsecond body\n"
func TestSplitsMessages(t *testing.T) {
ms := readAll(t, two, 0)
if len(ms) != 2 {
t.Fatalf("got %d messages", len(ms))
}
if string(ms[0].Data) != "From: alice@example.com\nSubject: one\n\nfirst body\n" {
t.Fatalf("first = %q", ms[0].Data)
}
if string(ms[1].Data) != "From: bob@example.com\nSubject: two\n\nsecond body\n" {
t.Fatalf("second = %q", ms[1].Data)
}
if ms[0].Envelope != "From alice@example.com Tue Jan 2 15:04:05 2024" {
t.Fatalf("envelope = %q", ms[0].Envelope)
}
}
func TestEnvelopeDate(t *testing.T) {
ms := readAll(t, two, 0)
want := time.Date(2024, 1, 2, 15, 4, 5, 0, time.UTC)
if got, ok := ms[0].Date(); !ok || !got.Equal(want) {
t.Fatalf("date = %v %v", got, ok)
}
for _, env := range []string{
"From x@y.z Tue Jan 2 15:04:05 +0200 2024",
"From x@y.z Tue Jan 2 15:04:05 2024",
"From x@y.z Tue Jan 2 15:04 2024",
"From MAILER-DAEMON Tue Jan 2 15:04:05 2024",
} {
if _, ok := (Message{Envelope: env}).Date(); !ok {
t.Errorf("%q must parse", env)
}
}
for _, env := range []string{"", "From x@y.z", "From x@y.z not a date"} {
if _, ok := (Message{Envelope: env}).Date(); ok {
t.Errorf("%q must not parse", env)
}
}
}
func TestProseLineStartingWithFromDoesNotSplit(t *testing.T) {
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nFrom now on we meet on Fridays.\nFrom: not a separator either\n"
ms := readAll(t, in, 0)
if len(ms) != 1 || !strings.Contains(string(ms[0].Data), "From now on we meet on Fridays.") {
t.Fatalf("messages: %d %q", len(ms), ms[0].Data)
}
}
func TestMboxrdUnescaping(t *testing.T) {
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n>From the log:\n>>From a quoted line\n> From not escaped\nplain\n"
ms := readAll(t, in, 0)
body := string(ms[0].Data)
if !strings.Contains(body, "\nFrom the log:\n") || !strings.Contains(body, "\n>From a quoted line\n") || !strings.Contains(body, "\n> From not escaped\n") {
t.Fatalf("body = %q", body)
}
}
func TestLeadingJunkAndEmptyInput(t *testing.T) {
if got := readAll(t, "", 0); len(got) != 0 {
t.Fatalf("empty input: %d", len(got))
}
if got := readAll(t, "\n\nsome preamble\nFrom a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nbody\n", 0); len(got) != 1 {
t.Fatalf("preamble must be ignored: %d", len(got))
}
if got := readAll(t, "just text, no separator at all\n", 0); len(got) != 0 {
t.Fatalf("no messages expected: %d", len(got))
}
}
func TestCRLFAndMissingFinalNewline(t *testing.T) {
in := "From a@x.com Tue Jan 2 15:04:05 2024\r\nSubject: s\r\n\r\nbody without newline"
ms := readAll(t, in, 0)
if len(ms) != 1 || !strings.HasSuffix(string(ms[0].Data), "body without newline") || !strings.Contains(string(ms[0].Data), "Subject: s\r\n") {
t.Fatalf("%q", ms[0].Data)
}
if string(readAll(t, "From a@x.com Tue Jan 2 15:04:05 2024\r\nSubject: one\r\n\r\nb\r\n\r\nFrom b@x.com Tue Jan 2 15:04:05 2024\r\nSubject: two\r\n\r\nb2\r\n", 0)[0].Data) != "Subject: one\r\n\r\nb\r\n" {
t.Fatal("the CRLF blank line before a separator belongs to the separator")
}
crlf := "From a@x.com Tue Jan 2 15:04:05 2024\r\nSubject: one\r\n\r\nb\r\n\r\nFrom b@x.com Tue Jan 2 15:04:05 2024\r\nSubject: two\r\n\r\nb2\r\n"
if got := readAll(t, crlf, 0); len(got) != 2 {
t.Fatalf("CRLF separators: %d", len(got))
}
}
func TestVeryLongLinesAreFine(t *testing.T) {
long := strings.Repeat("x", 300_000)
ms := readAll(t, "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n"+long+"\n", 0)
if len(ms) != 1 || !strings.Contains(string(ms[0].Data), long) {
t.Fatal("a long line must survive")
}
}
func TestOversizedMessagesAreReportedAndSkipped(t *testing.T) {
big := strings.Repeat("y\n", 5000)
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: ok1\n\nsmall\n" +
"From b@x.com Tue Jan 2 15:04:05 2024\nSubject: big\n\n" + big +
"From c@x.com Tue Jan 2 15:04:05 2024\nSubject: ok2\n\nsmall\n"
ms := readAll(t, in, 1000)
if len(ms) != 3 {
t.Fatalf("got %d", len(ms))
}
if ms[0].Err != nil || ms[2].Err != nil {
t.Fatalf("small messages must pass: %v %v", ms[0].Err, ms[2].Err)
}
if !errors.Is(ms[1].Err, ErrTooLarge) || ms[1].Data != nil {
t.Fatalf("big message: %v %d bytes", ms[1].Err, len(ms[1].Data))
}
if ms[1].Envelope != "From b@x.com Tue Jan 2 15:04:05 2024" || !strings.Contains(string(ms[2].Data), "ok2") {
t.Fatalf("the reader must resynchronise after a big message: %+v", ms[2])
}
}
func TestIndexIsOneBased(t *testing.T) {
ms := readAll(t, two, 0)
if ms[0].Index != 1 || ms[1].Index != 2 {
t.Fatalf("indexes: %d %d", ms[0].Index, ms[1].Index)
}
}
func TestReadErrorsArePropagated(t *testing.T) {
r := NewReader(io.MultiReader(strings.NewReader("From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nbody\n"), errReader{}), 0)
var last error
for i := 0; i < 5; i++ {
_, err := r.Next()
if err != nil {
last = err
break
}
}
if last == nil || last == io.EOF {
t.Fatalf("want the read error, got %v", last)
}
}
type errReader struct{}
func (errReader) Read([]byte) (int, error) { return 0, errors.New("disk on fire") }

View file

@ -2,6 +2,7 @@ package store
import (
"errors"
"fmt"
"strings"
"testing"
"time"
@ -227,3 +228,207 @@ func TestLongBodyIsStoredAsIs(t *testing.T) {
t.Fatalf("body len %d", len(got.Body))
}
}
func TestArchiveBatchInsertsDeduplicatesAndThreads(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
s.ArchiveMessage(l.ID, msg("<old@x>", "Already here", "x"))
reply := msg("<r@x>", "Re: Topic", "reply")
reply.InReplyTo = "<root@x>"
res, err := s.ArchiveBatch(l.ID, []Message{
msg("<root@x>", "Topic", "root"),
reply,
msg("<old@x>", "Already here", "x"), // duplicate of an existing message
msg("<root@x>", "Topic", "root"), // duplicate inside the batch
})
if err != nil {
t.Fatal(err)
}
if len(res) != 4 || !res[0].Created || !res[1].Created || res[2].Created || res[3].Created {
t.Fatalf("results: %+v", res)
}
if res[3].ID != res[0].ID {
t.Fatalf("a duplicate reports the id of the stored message: %+v", res)
}
r, _ := s.GetMessage(res[1].ID)
if r.ThreadID != res[0].ID {
t.Fatalf("threading inside a batch: thread %d, root %d", r.ThreadID, res[0].ID)
}
if _, total, _ := s.ListMessages(l.ID, 10, 0); total != 3 {
t.Fatalf("total = %d", total)
}
if got, _, _ := s.SearchMessages(l.ID, "reply", 10, 0); len(got) != 1 {
t.Fatal("batch inserts must be searchable")
}
}
func TestArchiveBatchIsAtomic(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
_, err := s.ArchiveBatch(l.ID, []Message{msg("<a@x>", "A", "a"), {Subject: "no id"}})
if err == nil {
t.Fatal("a message without an id must fail the batch")
}
if _, total, _ := s.ListMessages(l.ID, 10, 0); total != 0 {
t.Fatalf("nothing may be stored after a failed batch, got %d", total)
}
if res, err := s.ArchiveBatch(l.ID, nil); err != nil || len(res) != 0 {
t.Fatalf("empty batch: %v %v", res, err)
}
}
func threadOf(t *testing.T, s *Store, id int64) int64 {
t.Helper()
m, err := s.GetMessage(id)
if err != nil {
t.Fatal(err)
}
return m.ThreadID
}
func TestRethreadFixesMessagesImportedOutOfOrder(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
// Children first, root last: at insertion time none of the parents exists.
c2 := msg("<c2@x>", "Re: Re: T", "c2")
c2.InReplyTo = "<c1@x>"
c1 := msg("<c1@x>", "Re: T", "c1")
c1.InReplyTo = "<root@x>"
id2, _, _ := s.ArchiveMessage(l.ID, c2)
id1, _, _ := s.ArchiveMessage(l.ID, c1)
root, _, _ := s.ArchiveMessage(l.ID, msg("<root@x>", "T", "root"))
other, _, _ := s.ArchiveMessage(l.ID, msg("<other@x>", "Other", "o"))
if threadOf(t, s, id1) == root {
t.Fatal("precondition: the thread is broken before rethreading")
}
changed, err := s.Rethread(l.ID)
if err != nil {
t.Fatal(err)
}
if changed != 2 {
t.Fatalf("changed = %d, want the 2 broken messages", changed)
}
for _, id := range []int64{id1, id2, root} {
if got := threadOf(t, s, id); got != root {
t.Fatalf("message %d thread = %d, want %d", id, got, root)
}
}
if threadOf(t, s, other) != other {
t.Fatal("an unrelated message keeps its own thread")
}
if again, _ := s.Rethread(l.ID); again != 0 {
t.Fatalf("rethreading twice changes nothing, got %d", again)
}
if th, _ := s.ThreadMessages(l.ID, root); len(th) != 3 {
t.Fatalf("thread has %d messages", len(th))
}
}
func TestRethreadSurvivesCyclesAndSelfReplies(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
a := msg("<a@x>", "A", "a")
a.InReplyTo = "<b@x>"
b := msg("<b@x>", "B", "b")
b.InReplyTo = "<a@x>"
self := msg("<s@x>", "Self", "s")
self.InReplyTo = "<s@x>"
ia, _, _ := s.ArchiveMessage(l.ID, a)
ib, _, _ := s.ArchiveMessage(l.ID, b)
is, _, _ := s.ArchiveMessage(l.ID, self)
done := make(chan error, 1)
go func() { _, err := s.Rethread(l.ID); done <- err }()
select {
case err := <-done:
if err != nil {
t.Fatal(err)
}
case <-time.After(3 * time.Second):
t.Fatal("Rethread looped forever on a cycle")
}
if threadOf(t, s, ia) != threadOf(t, s, ib) {
t.Fatal("the two messages of a cycle share a thread")
}
if got := threadOf(t, s, ia); got != ia {
t.Fatalf("the cycle's thread is its smallest id (%d), got %d", ia, got)
}
if threadOf(t, s, is) != is {
t.Fatal("a message that replies to itself is its own thread")
}
}
func TestRethreadOnlyTouchesTheGivenList(t *testing.T) {
s := newTestStore(t)
a, _ := s.CreateList("a", "example.org", "A")
b, _ := s.CreateList("b", "example.org", "B")
child := msg("<c@x>", "Re", "c")
child.InReplyTo = "<p@x>"
cb, _, _ := s.ArchiveMessage(b.ID, child)
s.ArchiveMessage(b.ID, msg("<p@x>", "P", "p"))
// A parent with the same Message-ID in another list must not be used.
s.ArchiveMessage(a.ID, msg("<p@x>", "P in a", "p"))
if n, _ := s.Rethread(a.ID); n != 0 {
t.Fatalf("list a has nothing to fix, changed %d", n)
}
if threadOf(t, s, cb) != cb {
t.Fatal("list b must be untouched by rethreading list a")
}
if n, _ := s.Rethread(b.ID); n != 1 {
t.Fatalf("list b: changed %d", n)
}
if threadOf(t, s, cb) == cb {
t.Fatal("list b's child must now follow its parent")
}
}
func TestRethreadCycleRootIsTheSmallestIdEvenWhenEnteredFromElsewhere(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
// ids: x=1 replies to c; b=2 replies to c; c=3 replies to b. The cycle is {b, c}, entered from x at c,
// which is not its smallest id.
x := msg("<x@x>", "X", "x")
x.InReplyTo = "<c@x>"
b := msg("<b@x>", "B", "b")
b.InReplyTo = "<c@x>"
c := msg("<c@x>", "C", "c")
c.InReplyTo = "<b@x>"
ix, _, _ := s.ArchiveMessage(l.ID, x)
ib, _, _ := s.ArchiveMessage(l.ID, b)
ic, _, _ := s.ArchiveMessage(l.ID, c)
if _, err := s.Rethread(l.ID); err != nil {
t.Fatal(err)
}
want := min(ib, ic)
for _, id := range []int64{ix, ib, ic} {
if got := threadOf(t, s, id); got != want {
t.Fatalf("message %d: thread %d, want %d (the smallest id of the cycle)", id, got, want)
}
}
}
func TestRethreadIsLinearOnLongChains(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
const n = 20000
ms := make([]Message, n)
for i := range ms {
ms[i] = msg(fmt.Sprintf("<m%d@x>", i), "S", "b")
if i > 0 {
ms[i].InReplyTo = fmt.Sprintf("<m%d@x>", i-1)
}
}
if _, err := s.ArchiveBatch(l.ID, ms); err != nil {
t.Fatal(err)
}
done := make(chan struct{})
go func() { s.Rethread(l.ID); close(done) }()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("Rethread is not linear: a 20000-message chain took more than 5 seconds")
}
}

View file

@ -590,14 +590,52 @@ func scanMessage(row interface{ Scan(...any) error }, withBody bool) (Message, e
return m, err
}
// querier is what archiveOne needs from a *sql.DB or a *sql.Tx.
type querier interface {
Exec(query string, args ...any) (sql.Result, error)
QueryRow(query string, args ...any) *sql.Row
}
// ArchiveResult is the outcome for one message of ArchiveBatch.
type ArchiveResult struct {
ID int64
Created bool
}
// ArchiveMessage stores a post. A message already archived in the same list
// (same Message-ID) is not stored again: created is false and the existing id
// is returned, so MTA retries are harmless.
func (s *Store) ArchiveMessage(listID int64, m Message) (id int64, created bool, err error) {
return archiveOne(s.db, listID, m)
}
// ArchiveBatch stores many messages in one transaction: all or none. It is much
// faster than ArchiveMessage in a loop, because SQLite syncs once per commit.
func (s *Store) ArchiveBatch(listID int64, ms []Message) ([]ArchiveResult, error) {
if len(ms) == 0 {
return nil, nil
}
tx, err := s.db.Begin()
if err != nil {
return nil, err
}
defer tx.Rollback()
out := make([]ArchiveResult, 0, len(ms))
for _, m := range ms {
id, created, err := archiveOne(tx, listID, m)
if err != nil {
return nil, err
}
out = append(out, ArchiveResult{ID: id, Created: created})
}
return out, tx.Commit()
}
func archiveOne(q querier, listID int64, m Message) (id int64, created bool, err error) {
if m.MessageID == "" {
return 0, false, errors.New("message id is required")
}
err = s.db.QueryRow(`SELECT id FROM archive WHERE list_id=? AND message_id=?`, listID, m.MessageID).Scan(&id)
err = q.QueryRow(`SELECT id FROM archive WHERE list_id=? AND message_id=?`, listID, m.MessageID).Scan(&id)
if err == nil {
return id, false, nil
}
@ -606,12 +644,12 @@ func (s *Store) ArchiveMessage(listID int64, m Message) (id int64, created bool,
}
var thread int64
if m.InReplyTo != "" {
e := s.db.QueryRow(`SELECT thread_id FROM archive WHERE list_id=? AND message_id=?`, listID, m.InReplyTo).Scan(&thread)
e := q.QueryRow(`SELECT thread_id FROM archive WHERE list_id=? AND message_id=?`, listID, m.InReplyTo).Scan(&thread)
if e != nil && !errors.Is(e, sql.ErrNoRows) {
return 0, false, e
}
}
res, err := s.db.Exec(`INSERT INTO archive(list_id, thread_id, message_id, in_reply_to, subject, sender, date, body, size)
res, err := q.Exec(`INSERT INTO archive(list_id, thread_id, message_id, in_reply_to, subject, sender, date, body, size)
VALUES(?,?,?,?,?,?,?,?,?)`,
listID, thread, m.MessageID, m.InReplyTo, m.Subject, m.Sender, m.Date.Unix(), m.Body, len(m.Body))
if err != nil {
@ -619,13 +657,102 @@ func (s *Store) ArchiveMessage(listID int64, m Message) (id int64, created bool,
}
id, _ = res.LastInsertId()
if thread == 0 {
if _, err := s.db.Exec(`UPDATE archive SET thread_id=id WHERE id=?`, id); err != nil {
if _, err := q.Exec(`UPDATE archive SET thread_id=id WHERE id=?`, id); err != nil {
return 0, false, err
}
}
return id, true, nil
}
// Rethread recomputes the thread of every message of a list from the In-Reply-To
// links, and returns how many messages changed thread. Insertion order does not
// matter: a message imported before its parent ends up in the parent's thread.
// A cycle of replies becomes one thread whose id is the smallest id in the cycle.
func (s *Store) Rethread(listID int64) (int, error) {
rows, err := s.db.Query(`SELECT id, message_id, in_reply_to, thread_id FROM archive WHERE list_id=?`, listID)
if err != nil {
return 0, err
}
type row struct {
id, thread int64
replyTo string
}
var all []row
byMsgID := map[string]int64{}
for rows.Next() {
var r row
var msgID string
if err := rows.Scan(&r.id, &msgID, &r.replyTo, &r.thread); err != nil {
rows.Close()
return 0, err
}
all = append(all, r)
byMsgID[msgID] = r.id
}
rows.Close()
if err := rows.Err(); err != nil {
return 0, err
}
parent := map[int64]int64{}
for _, r := range all {
if p, ok := byMsgID[r.replyTo]; ok && r.replyTo != "" {
parent[r.id] = p
}
}
roots := map[int64]int64{}
root := func(start int64) int64 {
if r, ok := roots[start]; ok {
return r
}
path := []int64{start}
seen := map[int64]int{start: 0}
cur := start
var result int64
for {
p, ok := parent[cur]
if !ok {
result = cur
break
}
if r, known := roots[p]; known {
result = r
break
}
if at, loop := seen[p]; loop { // a cycle: its smallest id is the root
result = path[at]
for _, id := range path[at:] {
result = min(result, id)
}
break
}
seen[p] = len(path)
path = append(path, p)
cur = p
}
for _, id := range path {
roots[id] = result
}
return result
}
tx, err := s.db.Begin()
if err != nil {
return 0, err
}
defer tx.Rollback()
changed := 0
for _, r := range all {
if want := root(r.id); want != r.thread {
if _, err := tx.Exec(`UPDATE archive SET thread_id=? WHERE id=?`, want, r.id); err != nil {
return 0, err
}
changed++
}
}
return changed, tx.Commit()
}
// GetMessage returns an archived message with its body.
func (s *Store) GetMessage(id int64) (Message, error) {
return scanMessage(s.db.QueryRow(`SELECT `+msgCols+`, body FROM archive WHERE id=?`, id), true)