diff --git a/admin/src/lib/api.test.ts b/admin/src/lib/api.test.ts index c8622e6..02124df 100644 --- a/admin/src/lib/api.test.ts +++ b/admin/src/lib/api.test.ts @@ -159,4 +159,17 @@ describe('endpoints', () => { ['GET', '/api/v1/queue'], ]) }) + + it('uploads an mbox as the raw body with its own content type', async () => { + mockFetch(respond(200, { imported: 2, duplicates: 0, failed: 0, errors: [] })) + const file = new File(['From a@x.com Tue Jan 2 15:04:05 2024\nSubject: x\n\nbody\n'], 'dev.mbox') + const sum = await api.archive.import(7, file) + expect(sum).toEqual({ imported: 2, duplicates: 0, failed: 0, errors: [] }) + expect(calls[0].url).toBe('/api/v1/lists/7/archive/import') + expect(calls[0].init.method).toBe('POST') + expect(calls[0].init.body).toBe(file) // streamed as is, not JSON-encoded + const headers = calls[0].init.headers as Record + expect(headers['Content-Type']).toBe('application/mbox') + expect(headers.Authorization).toBe('Bearer secret-token') + }) }) diff --git a/admin/src/lib/api.ts b/admin/src/lib/api.ts index 23666fb..a33e798 100644 --- a/admin/src/lib/api.ts +++ b/admin/src/lib/api.ts @@ -42,6 +42,13 @@ export interface Message { body?: string } +export interface ImportSummary { + imported: number + duplicates: number + failed: number + errors: string[] +} + export interface ArchivePage { total: number messages: Message[] @@ -90,14 +97,16 @@ async function request(method: string, path: string, body?: unknown): Promise const headers: Record = {} const token = session.get() if (token) headers.Authorization = `Bearer ${token}` - if (body !== undefined) headers['Content-Type'] = 'application/json' + // A file is streamed as is; everything else is JSON. + const raw = body instanceof Blob + if (body !== undefined) headers['Content-Type'] = raw ? 'application/mbox' : 'application/json' let res: Response try { res = await fetch(`/api/v1${path}`, { method, headers, - body: body === undefined ? undefined : JSON.stringify(body), + body: body === undefined ? undefined : raw ? body : JSON.stringify(body), }) } catch { throw new ApiError(0, 'Cannot reach the server') @@ -159,6 +168,7 @@ export const api = { const s = qs.toString() return request('GET', `/lists/${listId}/archive${s ? `?${s}` : ''}`) }, + import: (listId: number, file: Blob) => request('POST', `/lists/${listId}/archive/import`, file), get: (id: number) => request('GET', `/archive/${id}`), remove: (id: number) => request('DELETE', `/archive/${id}`), }, diff --git a/admin/src/tabs/ArchiveTab.test.tsx b/admin/src/tabs/ArchiveTab.test.tsx index b255286..20360af 100644 --- a/admin/src/tabs/ArchiveTab.test.tsx +++ b/admin/src/tabs/ArchiveTab.test.tsx @@ -171,4 +171,78 @@ describe('archive tab', () => { expect(state.archive[1]).toHaveLength(2) }) }) + + describe('importing an mbox', () => { + const entry = (n: number) => + `From user${n}@x.com Tue Jan 2 15:04:05 2024\nFrom: User ${n} \nSubject: Imported ${n}\nMessage-ID: \n\nbody ${n}\n\n` + const mboxFile = (...ns: number[]) => new File([ns.map(entry).join('')], 'dev.mbox', { type: 'application/mbox' }) + + it('uploads the file, shows what happened and lists the new messages', async () => { + const { user, state } = await setup([]) + await screen.findByText(/no messages archived yet/i) + await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1, 2, 3)) + + const status = await screen.findByRole('status') + expect(status).toHaveTextContent('Imported 3 messages') + expect(state.importTypes).toEqual(['application/mbox']) + expect(state.archive[1]).toHaveLength(3) + expect(await screen.findByRole('button', { name: 'Imported 3' })).toBeInTheDocument() + expect(screen.getByRole('heading', { name: 'Archive (3)', level: 2 })).toBeInTheDocument() + }) + + it('says when messages were already there', async () => { + const { user } = await setup([]) + await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1, 2)) + await screen.findByText(/Imported 2 messages/) + await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1, 2, 3)) + const status = await screen.findByText(/Imported 1 message\b/) + expect(status).toHaveTextContent('2 already present') + }) + + it('lists the messages that could not be read', async () => { + const { user } = await setup([]) + const file = new File([entry(1) + 'From x@y.z Tue Jan 2 15:04:05 2024\n\nno headers here\n'], 'bad.mbox') + await user.upload(screen.getByLabelText(/mbox file/i), file) + const status = await screen.findByRole('status') + expect(status).toHaveTextContent('1 failed') + expect(within(status).getByText(/message 2: parse message: no headers/)).toBeInTheDocument() + }) + + it('shows a server error', async () => { + const { user, state } = await setup([]) + state.fail['POST /lists/1/archive/import'] = { status: 500, error: 'disk full' } + await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1)) + expect(await screen.findByRole('alert')).toHaveTextContent('disk full') + expect(screen.queryByRole('status')).not.toBeInTheDocument() + }) + + it('shows that it is working, and cannot be started twice', async () => { + let release!: () => void + const { user, state } = await setup([]) + state.hold = new Promise((resolve) => (release = resolve)) + await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1)) + const busy = await screen.findByRole('button', { name: /importing/i }) + expect(busy).toBeDisabled() + release() + expect(await screen.findByRole('status')).toHaveTextContent('Imported 1 message') + expect(screen.getByRole('button', { name: /^import mbox$/i })).toBeEnabled() + }) + + it('opens the file chooser from the button and accepts mbox-like files', async () => { + await setup([]) + const input = screen.getByLabelText(/mbox file/i) as HTMLInputElement + expect(input.type).toBe('file') + expect(input.accept).toMatch(/\.mbox/) + expect(screen.getByRole('button', { name: /^import mbox$/i })).toBeInTheDocument() + }) + + it('can import the same file again after a first attempt (the input is reset)', async () => { + const { user, state } = await setup([]) + const file = mboxFile(1) + await user.upload(screen.getByLabelText(/mbox file/i), file) + await screen.findByText(/Imported 1 message/) + await user.upload(screen.getByLabelText(/mbox file/i), file) + await waitFor(() => expect(state.importTypes).toHaveLength(2)) + }) + }) }) diff --git a/admin/src/tabs/ArchiveTab.tsx b/admin/src/tabs/ArchiveTab.tsx index 611ffa6..efcb1f8 100644 --- a/admin/src/tabs/ArchiveTab.tsx +++ b/admin/src/tabs/ArchiveTab.tsx @@ -1,22 +1,48 @@ import { keepPreviousData, useMutation, useQuery, useQueryClient } from '@tanstack/react-query' -import { useState, type FormEvent } from 'react' +import { useRef, useState, type ChangeEvent, type FormEvent } from 'react' import { useOutletContext } from 'react-router' import { toast } from 'sonner' import { Button } from '@/components/ui/button' import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from '@/components/ui/dialog' import { Input } from '@/components/ui/input' import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table' -import { api, type List } from '@/lib/api' +import { api, type ImportSummary, type List } from '@/lib/api' import { formatBytes, formatDate } from '@/lib/format' const PAGE_SIZE = 25 +function plural(n: number, one: string, many: string) { + return `${n} ${n === 1 ? one : many}` +} + export function ArchiveTab() { const list = useOutletContext() const [input, setInput] = useState('') const [q, setQ] = useState('') const [page, setPage] = useState(0) const [openId, setOpenId] = useState(null) + const queryClient = useQueryClient() + const fileInput = useRef(null) + const [imported, setImported] = useState(null) + const [importError, setImportError] = useState(null) + + const importMbox = useMutation({ + mutationFn: (file: File) => api.archive.import(list.id, file), + onSuccess: async (summary) => { + setImported(summary) + await queryClient.invalidateQueries({ queryKey: ['archive', list.id] }) + }, + onError: (e: Error) => setImportError(e.message), + }) + + function chooseFile(e: ChangeEvent) { + const file = e.target.files?.[0] + e.target.value = '' // lets the same file be chosen again + if (!file) return + setImported(null) + setImportError(null) + importMbox.mutate(file) + } const archive = useQuery({ queryKey: ['archive', list.id, q, page], @@ -43,7 +69,42 @@ export function ArchiveTab() { return (
-

Archive{archive.data && !q ? ` (${total})` : ''}

+
+

Archive{archive.data && !q ? ` (${total})` : ''}

+ + +
+ {importError && ( +

+ {importError} +

+ )} + {imported && ( +
+

+ Imported {plural(imported.imported, 'message', 'messages')} + {imported.duplicates > 0 && `, ${imported.duplicates} already present`} + {imported.failed > 0 && `, ${imported.failed} failed`} +

+ {imported.errors.length > 0 && ( +
    + {imported.errors.map((e) => ( +
  • {e}
  • + ))} + {imported.failed > imported.errors.length &&
  • … and {imported.failed - imported.errors.length} more
  • } +
+ )} +
+ )} {list.archive_policy === 'off' && (

Archiving is off for this list: new messages are not stored. Set Archive to private or public in Settings. diff --git a/admin/src/test/fakeApi.ts b/admin/src/test/fakeApi.ts index 4355086..fd93424 100644 --- a/admin/src/test/fakeApi.ts +++ b/admin/src/test/fakeApi.ts @@ -11,6 +11,10 @@ export interface FakeState { queue: QueueItem[] /** Like the server, the queue listing is capped. */ queueLimit: number + /** Content types seen on archive imports. */ + importTypes: string[] + /** When set, archive imports wait for it (to look at the busy state). */ + hold?: Promise nextId: number /** Every request seen, as "METHOD /path". */ calls: string[] @@ -19,7 +23,7 @@ export interface FakeState { } export function newState(overrides: Partial = {}): FakeState { - return { token: 'good', lists: [], members: {}, held: {}, archive: {}, queue: [], queueLimit: 100, nextId: 100, calls: [], fail: {}, ...overrides } + return { token: 'good', lists: [], members: {}, held: {}, archive: {}, queue: [], queueLimit: 100, importTypes: [], nextId: 100, calls: [], fail: {}, ...overrides } } export function aList(over: Partial = {}): List { @@ -62,7 +66,8 @@ export function installFakeApi(state: FakeState) { delete state.fail[key] return err(f.status, f.error) } - const body = init.body ? JSON.parse(init.body as string) : {} + // Files (archive imports) are not JSON. + const body = typeof init.body === 'string' && init.body ? JSON.parse(init.body) : {} let m: RegExpMatchArray | null if (path === '/queue' && method === 'GET') { @@ -132,6 +137,34 @@ export function installFakeApi(state: FakeState) { } return err(404, 'not found') } + if ((m = path.match(/^\/lists\/(\d+)\/archive\/import$/)) && method === 'POST') { + const id = Number(m[1]) + if (!state.lists.some((l) => l.id === id)) return err(404, 'not found') + if (state.hold) await state.hold + state.importTypes.push((init.headers as Record)['Content-Type'] ?? '') + const text = typeof init.body === 'string' ? init.body : await (init.body as Blob).text() + const messages = text.split(/^From \S+ .*$/m).slice(1) + const archive = (state.archive[id] ??= []) + let imported = 0 + let duplicates = 0 + const errors: string[] = [] + messages.forEach((raw, i) => { + const subject = /^Subject: (.*)$/m.exec(raw)?.[1] + const messageId = /^Message-ID: (.*)$/m.exec(raw)?.[1] + if (!subject || !messageId) { + errors.push(`message ${i + 1}: parse message: no headers`) + return + } + if (archive.some((a) => a.message_id === messageId)) { + duplicates++ + return + } + const nid = state.nextId++ + archive.push({ id: nid, thread_id: nid, message_id: messageId, subject, sender: 'Imported', date: '2024-01-01T00:00:00Z', size: raw.length }) + imported++ + }) + return json(200, { imported, duplicates, failed: errors.length, errors }) + } if ((m = path.match(/^\/lists\/(\d+)\/archive$/)) && method === 'GET') { const id = Number(m[1]) const q = (url.searchParams.get('q') ?? '').toLowerCase()