feat(admin): import an mbox from the Archive tab
- Add upload of an mbox file with a summary of imported, already present and failed messages - List the first failing messages and refresh the archive - Stream the file as the raw request body
This commit is contained in:
parent
cc47784fdb
commit
d8cc5ed68a
5 changed files with 198 additions and 7 deletions
|
|
@ -159,4 +159,17 @@ describe('endpoints', () => {
|
|||
['GET', '/api/v1/queue'],
|
||||
])
|
||||
})
|
||||
|
||||
it('uploads an mbox as the raw body with its own content type', async () => {
|
||||
mockFetch(respond(200, { imported: 2, duplicates: 0, failed: 0, errors: [] }))
|
||||
const file = new File(['From a@x.com Tue Jan 2 15:04:05 2024\nSubject: x\n\nbody\n'], 'dev.mbox')
|
||||
const sum = await api.archive.import(7, file)
|
||||
expect(sum).toEqual({ imported: 2, duplicates: 0, failed: 0, errors: [] })
|
||||
expect(calls[0].url).toBe('/api/v1/lists/7/archive/import')
|
||||
expect(calls[0].init.method).toBe('POST')
|
||||
expect(calls[0].init.body).toBe(file) // streamed as is, not JSON-encoded
|
||||
const headers = calls[0].init.headers as Record<string, string>
|
||||
expect(headers['Content-Type']).toBe('application/mbox')
|
||||
expect(headers.Authorization).toBe('Bearer secret-token')
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -42,6 +42,13 @@ export interface Message {
|
|||
body?: string
|
||||
}
|
||||
|
||||
export interface ImportSummary {
|
||||
imported: number
|
||||
duplicates: number
|
||||
failed: number
|
||||
errors: string[]
|
||||
}
|
||||
|
||||
export interface ArchivePage {
|
||||
total: number
|
||||
messages: Message[]
|
||||
|
|
@ -90,14 +97,16 @@ async function request<T>(method: string, path: string, body?: unknown): Promise
|
|||
const headers: Record<string, string> = {}
|
||||
const token = session.get()
|
||||
if (token) headers.Authorization = `Bearer ${token}`
|
||||
if (body !== undefined) headers['Content-Type'] = 'application/json'
|
||||
// A file is streamed as is; everything else is JSON.
|
||||
const raw = body instanceof Blob
|
||||
if (body !== undefined) headers['Content-Type'] = raw ? 'application/mbox' : 'application/json'
|
||||
|
||||
let res: Response
|
||||
try {
|
||||
res = await fetch(`/api/v1${path}`, {
|
||||
method,
|
||||
headers,
|
||||
body: body === undefined ? undefined : JSON.stringify(body),
|
||||
body: body === undefined ? undefined : raw ? body : JSON.stringify(body),
|
||||
})
|
||||
} catch {
|
||||
throw new ApiError(0, 'Cannot reach the server')
|
||||
|
|
@ -159,6 +168,7 @@ export const api = {
|
|||
const s = qs.toString()
|
||||
return request<ArchivePage>('GET', `/lists/${listId}/archive${s ? `?${s}` : ''}`)
|
||||
},
|
||||
import: (listId: number, file: Blob) => request<ImportSummary>('POST', `/lists/${listId}/archive/import`, file),
|
||||
get: (id: number) => request<Message>('GET', `/archive/${id}`),
|
||||
remove: (id: number) => request<void>('DELETE', `/archive/${id}`),
|
||||
},
|
||||
|
|
|
|||
|
|
@ -171,4 +171,78 @@ describe('archive tab', () => {
|
|||
expect(state.archive[1]).toHaveLength(2)
|
||||
})
|
||||
})
|
||||
|
||||
describe('importing an mbox', () => {
|
||||
const entry = (n: number) =>
|
||||
`From user${n}@x.com Tue Jan 2 15:04:05 2024\nFrom: User ${n} <user${n}@x.com>\nSubject: Imported ${n}\nMessage-ID: <imp${n}@x>\n\nbody ${n}\n\n`
|
||||
const mboxFile = (...ns: number[]) => new File([ns.map(entry).join('')], 'dev.mbox', { type: 'application/mbox' })
|
||||
|
||||
it('uploads the file, shows what happened and lists the new messages', async () => {
|
||||
const { user, state } = await setup([])
|
||||
await screen.findByText(/no messages archived yet/i)
|
||||
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1, 2, 3))
|
||||
|
||||
const status = await screen.findByRole('status')
|
||||
expect(status).toHaveTextContent('Imported 3 messages')
|
||||
expect(state.importTypes).toEqual(['application/mbox'])
|
||||
expect(state.archive[1]).toHaveLength(3)
|
||||
expect(await screen.findByRole('button', { name: 'Imported 3' })).toBeInTheDocument()
|
||||
expect(screen.getByRole('heading', { name: 'Archive (3)', level: 2 })).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('says when messages were already there', async () => {
|
||||
const { user } = await setup([])
|
||||
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1, 2))
|
||||
await screen.findByText(/Imported 2 messages/)
|
||||
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1, 2, 3))
|
||||
const status = await screen.findByText(/Imported 1 message\b/)
|
||||
expect(status).toHaveTextContent('2 already present')
|
||||
})
|
||||
|
||||
it('lists the messages that could not be read', async () => {
|
||||
const { user } = await setup([])
|
||||
const file = new File([entry(1) + 'From x@y.z Tue Jan 2 15:04:05 2024\n\nno headers here\n'], 'bad.mbox')
|
||||
await user.upload(screen.getByLabelText(/mbox file/i), file)
|
||||
const status = await screen.findByRole('status')
|
||||
expect(status).toHaveTextContent('1 failed')
|
||||
expect(within(status).getByText(/message 2: parse message: no headers/)).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('shows a server error', async () => {
|
||||
const { user, state } = await setup([])
|
||||
state.fail['POST /lists/1/archive/import'] = { status: 500, error: 'disk full' }
|
||||
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1))
|
||||
expect(await screen.findByRole('alert')).toHaveTextContent('disk full')
|
||||
expect(screen.queryByRole('status')).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('shows that it is working, and cannot be started twice', async () => {
|
||||
let release!: () => void
|
||||
const { user, state } = await setup([])
|
||||
state.hold = new Promise<void>((resolve) => (release = resolve))
|
||||
await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1))
|
||||
const busy = await screen.findByRole('button', { name: /importing/i })
|
||||
expect(busy).toBeDisabled()
|
||||
release()
|
||||
expect(await screen.findByRole('status')).toHaveTextContent('Imported 1 message')
|
||||
expect(screen.getByRole('button', { name: /^import mbox$/i })).toBeEnabled()
|
||||
})
|
||||
|
||||
it('opens the file chooser from the button and accepts mbox-like files', async () => {
|
||||
await setup([])
|
||||
const input = screen.getByLabelText(/mbox file/i) as HTMLInputElement
|
||||
expect(input.type).toBe('file')
|
||||
expect(input.accept).toMatch(/\.mbox/)
|
||||
expect(screen.getByRole('button', { name: /^import mbox$/i })).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('can import the same file again after a first attempt (the input is reset)', async () => {
|
||||
const { user, state } = await setup([])
|
||||
const file = mboxFile(1)
|
||||
await user.upload(screen.getByLabelText(/mbox file/i), file)
|
||||
await screen.findByText(/Imported 1 message/)
|
||||
await user.upload(screen.getByLabelText(/mbox file/i), file)
|
||||
await waitFor(() => expect(state.importTypes).toHaveLength(2))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,22 +1,48 @@
|
|||
import { keepPreviousData, useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
|
||||
import { useState, type FormEvent } from 'react'
|
||||
import { useRef, useState, type ChangeEvent, type FormEvent } from 'react'
|
||||
import { useOutletContext } from 'react-router'
|
||||
import { toast } from 'sonner'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from '@/components/ui/dialog'
|
||||
import { Input } from '@/components/ui/input'
|
||||
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table'
|
||||
import { api, type List } from '@/lib/api'
|
||||
import { api, type ImportSummary, type List } from '@/lib/api'
|
||||
import { formatBytes, formatDate } from '@/lib/format'
|
||||
|
||||
const PAGE_SIZE = 25
|
||||
|
||||
function plural(n: number, one: string, many: string) {
|
||||
return `${n} ${n === 1 ? one : many}`
|
||||
}
|
||||
|
||||
export function ArchiveTab() {
|
||||
const list = useOutletContext<List>()
|
||||
const [input, setInput] = useState('')
|
||||
const [q, setQ] = useState('')
|
||||
const [page, setPage] = useState(0)
|
||||
const [openId, setOpenId] = useState<number | null>(null)
|
||||
const queryClient = useQueryClient()
|
||||
const fileInput = useRef<HTMLInputElement>(null)
|
||||
const [imported, setImported] = useState<ImportSummary | null>(null)
|
||||
const [importError, setImportError] = useState<string | null>(null)
|
||||
|
||||
const importMbox = useMutation({
|
||||
mutationFn: (file: File) => api.archive.import(list.id, file),
|
||||
onSuccess: async (summary) => {
|
||||
setImported(summary)
|
||||
await queryClient.invalidateQueries({ queryKey: ['archive', list.id] })
|
||||
},
|
||||
onError: (e: Error) => setImportError(e.message),
|
||||
})
|
||||
|
||||
function chooseFile(e: ChangeEvent<HTMLInputElement>) {
|
||||
const file = e.target.files?.[0]
|
||||
e.target.value = '' // lets the same file be chosen again
|
||||
if (!file) return
|
||||
setImported(null)
|
||||
setImportError(null)
|
||||
importMbox.mutate(file)
|
||||
}
|
||||
|
||||
const archive = useQuery({
|
||||
queryKey: ['archive', list.id, q, page],
|
||||
|
|
@ -43,7 +69,42 @@ export function ArchiveTab() {
|
|||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<h2 className="text-lg font-semibold">Archive{archive.data && !q ? ` (${total})` : ''}</h2>
|
||||
<div className="flex items-center justify-between">
|
||||
<h2 className="text-lg font-semibold">Archive{archive.data && !q ? ` (${total})` : ''}</h2>
|
||||
<input
|
||||
ref={fileInput}
|
||||
type="file"
|
||||
accept=".mbox,.mbx,.txt,application/mbox,text/plain"
|
||||
aria-label="Mbox file"
|
||||
className="hidden"
|
||||
onChange={chooseFile}
|
||||
/>
|
||||
<Button variant="outline" disabled={importMbox.isPending} onClick={() => fileInput.current?.click()}>
|
||||
{importMbox.isPending ? 'Importing…' : 'Import mbox'}
|
||||
</Button>
|
||||
</div>
|
||||
{importError && (
|
||||
<p role="alert" className="text-sm text-destructive">
|
||||
{importError}
|
||||
</p>
|
||||
)}
|
||||
{imported && (
|
||||
<div role="status" className="space-y-1 rounded-md border p-3 text-sm">
|
||||
<p>
|
||||
Imported {plural(imported.imported, 'message', 'messages')}
|
||||
{imported.duplicates > 0 && `, ${imported.duplicates} already present`}
|
||||
{imported.failed > 0 && `, ${imported.failed} failed`}
|
||||
</p>
|
||||
{imported.errors.length > 0 && (
|
||||
<ul className="list-disc pl-5 text-destructive">
|
||||
{imported.errors.map((e) => (
|
||||
<li key={e}>{e}</li>
|
||||
))}
|
||||
{imported.failed > imported.errors.length && <li>… and {imported.failed - imported.errors.length} more</li>}
|
||||
</ul>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{list.archive_policy === 'off' && (
|
||||
<p className="rounded-md border p-3 text-sm text-muted-foreground">
|
||||
Archiving is off for this list: new messages are not stored. Set Archive to private or public in Settings.
|
||||
|
|
|
|||
|
|
@ -11,6 +11,10 @@ export interface FakeState {
|
|||
queue: QueueItem[]
|
||||
/** Like the server, the queue listing is capped. */
|
||||
queueLimit: number
|
||||
/** Content types seen on archive imports. */
|
||||
importTypes: string[]
|
||||
/** When set, archive imports wait for it (to look at the busy state). */
|
||||
hold?: Promise<void>
|
||||
nextId: number
|
||||
/** Every request seen, as "METHOD /path". */
|
||||
calls: string[]
|
||||
|
|
@ -19,7 +23,7 @@ export interface FakeState {
|
|||
}
|
||||
|
||||
export function newState(overrides: Partial<FakeState> = {}): FakeState {
|
||||
return { token: 'good', lists: [], members: {}, held: {}, archive: {}, queue: [], queueLimit: 100, nextId: 100, calls: [], fail: {}, ...overrides }
|
||||
return { token: 'good', lists: [], members: {}, held: {}, archive: {}, queue: [], queueLimit: 100, importTypes: [], nextId: 100, calls: [], fail: {}, ...overrides }
|
||||
}
|
||||
|
||||
export function aList(over: Partial<List> = {}): List {
|
||||
|
|
@ -62,7 +66,8 @@ export function installFakeApi(state: FakeState) {
|
|||
delete state.fail[key]
|
||||
return err(f.status, f.error)
|
||||
}
|
||||
const body = init.body ? JSON.parse(init.body as string) : {}
|
||||
// Files (archive imports) are not JSON.
|
||||
const body = typeof init.body === 'string' && init.body ? JSON.parse(init.body) : {}
|
||||
let m: RegExpMatchArray | null
|
||||
|
||||
if (path === '/queue' && method === 'GET') {
|
||||
|
|
@ -132,6 +137,34 @@ export function installFakeApi(state: FakeState) {
|
|||
}
|
||||
return err(404, 'not found')
|
||||
}
|
||||
if ((m = path.match(/^\/lists\/(\d+)\/archive\/import$/)) && method === 'POST') {
|
||||
const id = Number(m[1])
|
||||
if (!state.lists.some((l) => l.id === id)) return err(404, 'not found')
|
||||
if (state.hold) await state.hold
|
||||
state.importTypes.push((init.headers as Record<string, string>)['Content-Type'] ?? '')
|
||||
const text = typeof init.body === 'string' ? init.body : await (init.body as Blob).text()
|
||||
const messages = text.split(/^From \S+ .*$/m).slice(1)
|
||||
const archive = (state.archive[id] ??= [])
|
||||
let imported = 0
|
||||
let duplicates = 0
|
||||
const errors: string[] = []
|
||||
messages.forEach((raw, i) => {
|
||||
const subject = /^Subject: (.*)$/m.exec(raw)?.[1]
|
||||
const messageId = /^Message-ID: (.*)$/m.exec(raw)?.[1]
|
||||
if (!subject || !messageId) {
|
||||
errors.push(`message ${i + 1}: parse message: no headers`)
|
||||
return
|
||||
}
|
||||
if (archive.some((a) => a.message_id === messageId)) {
|
||||
duplicates++
|
||||
return
|
||||
}
|
||||
const nid = state.nextId++
|
||||
archive.push({ id: nid, thread_id: nid, message_id: messageId, subject, sender: 'Imported', date: '2024-01-01T00:00:00Z', size: raw.length })
|
||||
imported++
|
||||
})
|
||||
return json(200, { imported, duplicates, failed: errors.length, errors })
|
||||
}
|
||||
if ((m = path.match(/^\/lists\/(\d+)\/archive$/)) && method === 'GET') {
|
||||
const id = Number(m[1])
|
||||
const q = (url.searchParams.get('q') ?? '').toLowerCase()
|
||||
|
|
|
|||
Loading…
Reference in a new issue