feat(admin): import an mbox from the Archive tab

- Add upload of an mbox file with a summary of imported, already present and failed messages
- List the first failing messages and refresh the archive
- Stream the file as the raw request body
This commit is contained in:
tgiachi 2026-11-09 19:45:48 +01:00
commit d8cc5ed68a
5 changed files with 198 additions and 7 deletions

View file

@ -159,4 +159,17 @@ describe('endpoints', () => {
['GET', '/api/v1/queue'],
])
})
it('uploads an mbox as the raw body with its own content type', async () => {
mockFetch(respond(200, { imported: 2, duplicates: 0, failed: 0, errors: [] }))
const file = new File(['From a@x.com Tue Jan 2 15:04:05 2024\nSubject: x\n\nbody\n'], 'dev.mbox')
const sum = await api.archive.import(7, file)
expect(sum).toEqual({ imported: 2, duplicates: 0, failed: 0, errors: [] })
expect(calls[0].url).toBe('/api/v1/lists/7/archive/import')
expect(calls[0].init.method).toBe('POST')
expect(calls[0].init.body).toBe(file) // streamed as is, not JSON-encoded
const headers = calls[0].init.headers as Record<string, string>
expect(headers['Content-Type']).toBe('application/mbox')
expect(headers.Authorization).toBe('Bearer secret-token')
})
})

View file

@ -42,6 +42,13 @@ export interface Message {
body?: string
}
export interface ImportSummary {
imported: number
duplicates: number
failed: number
errors: string[]
}
export interface ArchivePage {
total: number
messages: Message[]
@ -90,14 +97,16 @@ async function request<T>(method: string, path: string, body?: unknown): Promise
const headers: Record<string, string> = {}
const token = session.get()
if (token) headers.Authorization = `Bearer ${token}`
if (body !== undefined) headers['Content-Type'] = 'application/json'
// A file is streamed as is; everything else is JSON.
const raw = body instanceof Blob
if (body !== undefined) headers['Content-Type'] = raw ? 'application/mbox' : 'application/json'
let res: Response
try {
res = await fetch(`/api/v1${path}`, {
method,
headers,
body: body === undefined ? undefined : JSON.stringify(body),
body: body === undefined ? undefined : raw ? body : JSON.stringify(body),
})
} catch {
throw new ApiError(0, 'Cannot reach the server')
@ -159,6 +168,7 @@ export const api = {
const s = qs.toString()
return request<ArchivePage>('GET', `/lists/${listId}/archive${s ? `?${s}` : ''}`)
},
import: (listId: number, file: Blob) => request<ImportSummary>('POST', `/lists/${listId}/archive/import`, file),
get: (id: number) => request<Message>('GET', `/archive/${id}`),
remove: (id: number) => request<void>('DELETE', `/archive/${id}`),
},

View file

@ -171,4 +171,78 @@ describe('archive tab', () => {
expect(state.archive[1]).toHaveLength(2)
})
})
describe('importing an mbox', () => {
const entry = (n: number) =>
`From user${n}@x.com Tue Jan 2 15:04:05 2024\nFrom: User ${n} <user${n}@x.com>\nSubject: Imported ${n}\nMessage-ID: <imp${n}@x>\n\nbody ${n}\n\n`
const mboxFile = (...ns: number[]) => new File([ns.map(entry).join('')], 'dev.mbox', { type: 'application/mbox' })
it('uploads the file, shows what happened and lists the new messages', async () => {
const { user, state } = await setup([])
await screen.findByText(/no messages archived yet/i)
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1, 2, 3))
const status = await screen.findByRole('status')
expect(status).toHaveTextContent('Imported 3 messages')
expect(state.importTypes).toEqual(['application/mbox'])
expect(state.archive[1]).toHaveLength(3)
expect(await screen.findByRole('button', { name: 'Imported 3' })).toBeInTheDocument()
expect(screen.getByRole('heading', { name: 'Archive (3)', level: 2 })).toBeInTheDocument()
})
it('says when messages were already there', async () => {
const { user } = await setup([])
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1, 2))
await screen.findByText(/Imported 2 messages/)
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1, 2, 3))
const status = await screen.findByText(/Imported 1 message\b/)
expect(status).toHaveTextContent('2 already present')
})
it('lists the messages that could not be read', async () => {
const { user } = await setup([])
const file = new File([entry(1) + 'From x@y.z Tue Jan 2 15:04:05 2024\n\nno headers here\n'], 'bad.mbox')
await user.upload(screen.getByLabelText(/mbox file/i), file)
const status = await screen.findByRole('status')
expect(status).toHaveTextContent('1 failed')
expect(within(status).getByText(/message 2: parse message: no headers/)).toBeInTheDocument()
})
it('shows a server error', async () => {
const { user, state } = await setup([])
state.fail['POST /lists/1/archive/import'] = { status: 500, error: 'disk full' }
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1))
expect(await screen.findByRole('alert')).toHaveTextContent('disk full')
expect(screen.queryByRole('status')).not.toBeInTheDocument()
})
it('shows that it is working, and cannot be started twice', async () => {
let release!: () => void
const { user, state } = await setup([])
state.hold = new Promise<void>((resolve) => (release = resolve))
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1))
const busy = await screen.findByRole('button', { name: /importing/i })
expect(busy).toBeDisabled()
release()
expect(await screen.findByRole('status')).toHaveTextContent('Imported 1 message')
expect(screen.getByRole('button', { name: /^import mbox$/i })).toBeEnabled()
})
it('opens the file chooser from the button and accepts mbox-like files', async () => {
await setup([])
const input = screen.getByLabelText(/mbox file/i) as HTMLInputElement
expect(input.type).toBe('file')
expect(input.accept).toMatch(/\.mbox/)
expect(screen.getByRole('button', { name: /^import mbox$/i })).toBeInTheDocument()
})
it('can import the same file again after a first attempt (the input is reset)', async () => {
const { user, state } = await setup([])
const file = mboxFile(1)
await user.upload(screen.getByLabelText(/mbox file/i), file)
await screen.findByText(/Imported 1 message/)
await user.upload(screen.getByLabelText(/mbox file/i), file)
await waitFor(() => expect(state.importTypes).toHaveLength(2))
})
})
})

View file

@ -1,22 +1,48 @@
import { keepPreviousData, useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
import { useState, type FormEvent } from 'react'
import { useRef, useState, type ChangeEvent, type FormEvent } from 'react'
import { useOutletContext } from 'react-router'
import { toast } from 'sonner'
import { Button } from '@/components/ui/button'
import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from '@/components/ui/dialog'
import { Input } from '@/components/ui/input'
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table'
import { api, type List } from '@/lib/api'
import { api, type ImportSummary, type List } from '@/lib/api'
import { formatBytes, formatDate } from '@/lib/format'
const PAGE_SIZE = 25
function plural(n: number, one: string, many: string) {
return `${n} ${n === 1 ? one : many}`
}
export function ArchiveTab() {
const list = useOutletContext<List>()
const [input, setInput] = useState('')
const [q, setQ] = useState('')
const [page, setPage] = useState(0)
const [openId, setOpenId] = useState<number | null>(null)
const queryClient = useQueryClient()
const fileInput = useRef<HTMLInputElement>(null)
const [imported, setImported] = useState<ImportSummary | null>(null)
const [importError, setImportError] = useState<string | null>(null)
const importMbox = useMutation({
mutationFn: (file: File) => api.archive.import(list.id, file),
onSuccess: async (summary) => {
setImported(summary)
await queryClient.invalidateQueries({ queryKey: ['archive', list.id] })
},
onError: (e: Error) => setImportError(e.message),
})
function chooseFile(e: ChangeEvent<HTMLInputElement>) {
const file = e.target.files?.[0]
e.target.value = '' // lets the same file be chosen again
if (!file) return
setImported(null)
setImportError(null)
importMbox.mutate(file)
}
const archive = useQuery({
queryKey: ['archive', list.id, q, page],
@ -43,7 +69,42 @@ export function ArchiveTab() {
return (
<div className="space-y-4">
<h2 className="text-lg font-semibold">Archive{archive.data && !q ? ` (${total})` : ''}</h2>
<div className="flex items-center justify-between">
<h2 className="text-lg font-semibold">Archive{archive.data && !q ? ` (${total})` : ''}</h2>
<input
ref={fileInput}
type="file"
accept=".mbox,.mbx,.txt,application/mbox,text/plain"
aria-label="Mbox file"
className="hidden"
onChange={chooseFile}
/>
<Button variant="outline" disabled={importMbox.isPending} onClick={() => fileInput.current?.click()}>
{importMbox.isPending ? 'Importing…' : 'Import mbox'}
</Button>
</div>
{importError && (
<p role="alert" className="text-sm text-destructive">
{importError}
</p>
)}
{imported && (
<div role="status" className="space-y-1 rounded-md border p-3 text-sm">
<p>
Imported {plural(imported.imported, 'message', 'messages')}
{imported.duplicates > 0 && `, ${imported.duplicates} already present`}
{imported.failed > 0 && `, ${imported.failed} failed`}
</p>
{imported.errors.length > 0 && (
<ul className="list-disc pl-5 text-destructive">
{imported.errors.map((e) => (
<li key={e}>{e}</li>
))}
{imported.failed > imported.errors.length && <li>… and {imported.failed - imported.errors.length} more</li>}
</ul>
)}
</div>
)}
{list.archive_policy === 'off' && (
<p className="rounded-md border p-3 text-sm text-muted-foreground">
Archiving is off for this list: new messages are not stored. Set Archive to private or public in Settings.

View file

@ -11,6 +11,10 @@ export interface FakeState {
queue: QueueItem[]
/** Like the server, the queue listing is capped. */
queueLimit: number
/** Content types seen on archive imports. */
importTypes: string[]
/** When set, archive imports wait for it (to look at the busy state). */
hold?: Promise<void>
nextId: number
/** Every request seen, as "METHOD /path". */
calls: string[]
@ -19,7 +23,7 @@ export interface FakeState {
}
export function newState(overrides: Partial<FakeState> = {}): FakeState {
return { token: 'good', lists: [], members: {}, held: {}, archive: {}, queue: [], queueLimit: 100, nextId: 100, calls: [], fail: {}, ...overrides }
return { token: 'good', lists: [], members: {}, held: {}, archive: {}, queue: [], queueLimit: 100, importTypes: [], nextId: 100, calls: [], fail: {}, ...overrides }
}
export function aList(over: Partial<List> = {}): List {
@ -62,7 +66,8 @@ export function installFakeApi(state: FakeState) {
delete state.fail[key]
return err(f.status, f.error)
}
const body = init.body ? JSON.parse(init.body as string) : {}
// Files (archive imports) are not JSON.
const body = typeof init.body === 'string' && init.body ? JSON.parse(init.body) : {}
let m: RegExpMatchArray | null
if (path === '/queue' && method === 'GET') {
@ -132,6 +137,34 @@ export function installFakeApi(state: FakeState) {
}
return err(404, 'not found')
}
if ((m = path.match(/^\/lists\/(\d+)\/archive\/import$/)) && method === 'POST') {
const id = Number(m[1])
if (!state.lists.some((l) => l.id === id)) return err(404, 'not found')
if (state.hold) await state.hold
state.importTypes.push((init.headers as Record<string, string>)['Content-Type'] ?? '')
const text = typeof init.body === 'string' ? init.body : await (init.body as Blob).text()
const messages = text.split(/^From \S+ .*$/m).slice(1)
const archive = (state.archive[id] ??= [])
let imported = 0
let duplicates = 0
const errors: string[] = []
messages.forEach((raw, i) => {
const subject = /^Subject: (.*)$/m.exec(raw)?.[1]
const messageId = /^Message-ID: (.*)$/m.exec(raw)?.[1]
if (!subject || !messageId) {
errors.push(`message ${i + 1}: parse message: no headers`)
return
}
if (archive.some((a) => a.message_id === messageId)) {
duplicates++
return
}
const nid = state.nextId++
archive.push({ id: nid, thread_id: nid, message_id: messageId, subject, sender: 'Imported', date: '2024-01-01T00:00:00Z', size: raw.length })
imported++
})
return json(200, { imported, duplicates, failed: errors.length, errors })
}
if ((m = path.match(/^\/lists\/(\d+)\/archive$/)) && method === 'GET') {
const id = Number(m[1])
const q = (url.searchParams.get('q') ?? '').toLowerCase()