feat: show failed deliveries in the API, CLI and UI

This commit is contained in:
tgiachi 2026-11-09 20:34:37 +01:00
commit afd067688c
9 changed files with 109 additions and 14 deletions

View file

@ -63,11 +63,14 @@ export interface QueueItem {
last_error: string
next_attempt: string
created_at: string
/** Given up for good: never retried, kept for a week so the admin can see it. */
failed: boolean
}
export interface Queue {
pending: number
retrying: number
failed: number
oldest: string | null
items: QueueItem[]
}

View file

@ -11,6 +11,7 @@ const item = (id: number, over: Partial<QueueItem> = {}): QueueItem => ({
last_error: '',
next_attempt: '2030-01-02T03:04:05Z',
created_at: '2030-01-01T00:00:00Z',
failed: false,
...over,
})
@ -51,7 +52,7 @@ describe('queue page', () => {
expect(within(fresh).getByText('—')).toBeInTheDocument()
expect(screen.getByText('Pending').nextElementSibling).toHaveTextContent('2')
expect(screen.getByText('Retrying').nextElementSibling).toHaveTextContent('1')
expect(screen.getByText('Retrying', { selector: 'div' }).nextElementSibling).toHaveTextContent('1')
expect(screen.getByText('Oldest').nextElementSibling).toHaveTextContent('2030-01-01 00:00 UTC')
})
@ -68,6 +69,26 @@ describe('queue page', () => {
expect(screen.queryByText(/showing/i)).not.toBeInTheDocument()
})
it('shows failed deliveries apart from pending ones, with the reason', async () => {
await setup([
item(1),
item(2, { attempts: 2, last_error: 'connection refused' }),
item(3, { attempts: 1, failed: true, last_error: 'refused by the relay: 554 policy rejection' }),
])
const failed = (await screen.findByText('user3@example.com')).closest('tr')!
expect(within(failed).getByText('Failed')).toBeInTheDocument()
expect(within(failed).getByText(/refused by the relay: 554 policy rejection/)).toBeInTheDocument()
expect(within(failed).getByText('—', { selector: 'td' })).toBeInTheDocument() // never retried: no next attempt
expect(within(screen.getByText('user2@example.com').closest('tr')!).getByText('Retrying')).toBeInTheDocument()
expect(within(screen.getByText('user1@example.com').closest('tr')!).getByText('Waiting')).toBeInTheDocument()
expect(screen.getByText('Pending').nextElementSibling).toHaveTextContent('2')
expect(screen.getByText('Retrying', { selector: 'div' }).nextElementSibling).toHaveTextContent('1')
expect(screen.getByText('Failed', { selector: 'div' }).nextElementSibling).toHaveTextContent('1')
const rows = screen.getAllByRole('row').slice(1).map((r) => r.textContent ?? '')
expect(rows[0]).toContain('user3@example.com') // failed first
})
it('refreshes on demand', async () => {
const { user, state } = await setup([item(1)])
await screen.findByText('user1@example.com')

View file

@ -10,6 +10,7 @@ import {
AlertDialogHeader,
AlertDialogTitle,
} from '@/components/ui/alert-dialog'
import { Badge } from '@/components/ui/badge'
import { Button } from '@/components/ui/button'
import { Card, CardContent } from '@/components/ui/card'
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table'
@ -38,9 +39,10 @@ export function QueuePage() {
{q && (
<>
<div className="grid gap-4 sm:grid-cols-3">
<div className="grid gap-4 sm:grid-cols-4">
<Stat label="Pending" value={String(q.pending)} />
<Stat label="Retrying" value={String(q.retrying)} />
<Stat label="Failed" value={String(q.failed)} />
<Stat label="Oldest" value={q.oldest ? formatDate(q.oldest) : '—'} />
</div>
{q.items.length === 0 ? (
@ -50,6 +52,7 @@ export function QueuePage() {
<TableHeader>
<TableRow>
<TableHead>Recipient</TableHead>
<TableHead>State</TableHead>
<TableHead>Attempts</TableHead>
<TableHead>Last error</TableHead>
<TableHead>Next attempt</TableHead>
@ -60,9 +63,18 @@ export function QueuePage() {
{q.items.map((i) => (
<TableRow key={i.id}>
<TableCell className="font-medium">{i.recipient}</TableCell>
<TableCell>
{i.failed ? (
<Badge variant="destructive">Failed</Badge>
) : i.attempts > 0 ? (
<Badge variant="secondary">Retrying</Badge>
) : (
<Badge variant="outline">Waiting</Badge>
)}
</TableCell>
<TableCell>{i.attempts}</TableCell>
<TableCell>{i.last_error || '—'}</TableCell>
<TableCell>{formatDate(i.next_attempt)}</TableCell>
<TableCell className="max-w-xs whitespace-normal">{i.last_error || '—'}</TableCell>
<TableCell>{i.failed ? '—' : formatDate(i.next_attempt)}</TableCell>
<TableCell className="text-right">
<Button size="sm" variant="outline" onClick={() => setDiscarding(i)}>
Discard

View file

@ -71,10 +71,12 @@ export function installFakeApi(state: FakeState) {
let m: RegExpMatchArray | null
if (path === '/queue' && method === 'GET') {
const items = [...state.queue].sort((a, b) => b.attempts - a.attempts).slice(0, state.queueLimit)
const retrying = state.queue.filter((i) => i.attempts > 0).length
const oldest = state.queue.map((i) => i.created_at).sort()[0] ?? null
return json(200, { pending: state.queue.length, retrying, oldest, items })
const items = [...state.queue].sort((a, b) => Number(b.failed) - Number(a.failed) || b.attempts - a.attempts).slice(0, state.queueLimit)
const waiting = state.queue.filter((i) => !i.failed)
const retrying = waiting.filter((i) => i.attempts > 0).length
const failed = state.queue.length - waiting.length
const oldest = waiting.map((i) => i.created_at).sort()[0] ?? null
return json(200, { pending: waiting.length, retrying, failed, oldest, items })
}
if ((m = path.match(/^\/queue\/(\d+)$/)) && method === 'DELETE') {
const id = Number(m[1])

View file

@ -44,6 +44,7 @@ components:
last_error: {type: string}
next_attempt: {type: string, format: date-time}
created_at: {type: string, format: date-time}
failed: {type: boolean, description: Given up for good (refused by the relay, or too old); never retried, listed for a week}
Member:
type: object
properties:
@ -264,7 +265,8 @@ paths:
type: object
properties:
pending: {type: integer}
retrying: {type: integer, description: Deliveries that failed at least once}
retrying: {type: integer, description: Waiting deliveries that failed at least once}
failed: {type: integer, description: Deliveries given up for good, kept for a week}
oldest: {type: string, format: date-time, nullable: true}
items: {type: array, items: {$ref: '#/components/schemas/QueueItem'}}
/queue/{id}:

View file

@ -523,6 +523,7 @@ type queuedDTO struct {
LastError string `json:"last_error"`
NextAttempt time.Time `json:"next_attempt"`
CreatedAt time.Time `json:"created_at"`
Failed bool `json:"failed"` // given up for good: never retried
}
func (s *server) getQueue(w http.ResponseWriter, _ *http.Request) {
@ -539,14 +540,14 @@ func (s *server) getQueue(w http.ResponseWriter, _ *http.Request) {
out := make([]queuedDTO, 0, len(items))
for _, it := range items {
out = append(out, queuedDTO{ID: it.ID, Recipient: it.Rcpt, Attempts: it.Attempts, LastError: it.LastError,
NextAttempt: it.NextAt, CreatedAt: it.CreatedAt})
NextAttempt: it.NextAt, CreatedAt: it.CreatedAt, Failed: it.Failed})
}
var oldest *time.Time
if !st.Oldest.IsZero() {
oldest = &st.Oldest
}
writeJSON(w, http.StatusOK, map[string]any{
"pending": st.Pending, "retrying": st.Retrying, "oldest": oldest, "items": out,
"pending": st.Pending, "retrying": st.Retrying, "failed": st.Failed, "oldest": oldest, "items": out,
})
}

View file

@ -7,6 +7,7 @@ import (
"fmt"
"net/http"
"net/http/httptest"
"strconv"
"strings"
"testing"
"time"
@ -448,6 +449,7 @@ func queueServer(t *testing.T) (http.Handler, *store.Store) {
type queueJSON struct {
Pending int `json:"pending"`
Retrying int `json:"retrying"`
Failed int `json:"failed"`
Oldest *time.Time `json:"oldest"`
Items []struct {
ID int64 `json:"id"`
@ -456,6 +458,7 @@ type queueJSON struct {
LastError string `json:"last_error"`
NextAttempt time.Time `json:"next_attempt"`
CreatedAt time.Time `json:"created_at"`
Failed bool `json:"failed"`
} `json:"items"`
}
@ -630,3 +633,24 @@ func TestArchiveImportStoppedByAReadErrorIsAnErrorResponse(t *testing.T) {
t.Fatalf("the first message was complete and is kept, got %d", total)
}
}
func TestQueueShowsFailedDeliveriesApartFromPendingOnes(t *testing.T) {
h, s := queueServer(t)
s.Enqueue([]byte("x"), []store.NewOutbound{{From: "b@x.org", Rcpt: "a@x.com"}, {From: "b@x.org", Rcpt: "b@x.com"}, {From: "b@x.org", Rcpt: "c@x.com"}})
due, _ := s.DueOutbox(10)
s.FailOutbox(due[2].ID, "refused by the relay: 554 no")
s.RetryOutbox(due[1].ID, time.Now().Add(time.Hour), "down")
var q queueJSON
decode(t, do(t, h, "GET", "/api/v1/queue", "", true), &q)
if q.Pending != 2 || q.Retrying != 1 || q.Failed != 1 || len(q.Items) != 3 {
t.Fatalf("queue: %+v", q)
}
if !q.Items[0].Failed || q.Items[0].Recipient != "c@x.com" || q.Items[0].LastError != "refused by the relay: 554 no" {
t.Fatalf("failed deliveries come first, with the reason: %+v", q.Items[0])
}
if q.Items[1].Failed || q.Items[2].Failed {
t.Fatalf("only one failed: %+v", q.Items)
}
expect(t, do(t, h, "DELETE", "/api/v1/queue/"+strconv.FormatInt(due[2].ID, 10), "", true), 204)
}

View file

@ -776,3 +776,24 @@ func TestDKIMDomainIsLowercased(t *testing.T) {
t.Fatalf("lookup name: %v", *asked)
}
}
func TestQueueShowsFailedDeliveries(t *testing.T) {
e := newEnv(t)
e.s.Enqueue([]byte("x"), []store.NewOutbound{{From: "b@x.org", Rcpt: "waits@x.com"}, {From: "b@x.org", Rcpt: "retries@x.com"}, {From: "b@x.org", Rcpt: "lost@x.com"}})
due, _ := e.s.DueOutbox(10)
e.s.RetryOutbox(due[1].ID, time.Now().Add(time.Hour), "connection refused")
e.s.FailOutbox(due[2].ID, "refused by the relay: 554 policy rejection")
out := e.ok(t, "queue", "ls")
for _, want := range []string{"Pending: 2", "Retrying: 1", "Failed: 1", "STATE", "failed", "retrying", "waiting", "refused by the relay: 554 policy rejection"} {
if !strings.Contains(out, want) {
t.Fatalf("queue ls lacks %q:\n%s", want, out)
}
}
if strings.Index(out, "lost@x.com") > strings.Index(out, "retries@x.com") || strings.Index(out, "retries@x.com") > strings.Index(out, "waits@x.com") {
t.Fatalf("failed first, then retrying, then waiting:\n%s", out)
}
e.ok(t, "queue", "rm", fmt.Sprint(due[2].ID))
if out := e.ok(t, "queue", "ls"); strings.Contains(out, "lost@x.com") || !strings.Contains(out, "Failed: 0") {
t.Fatalf("a failed delivery can be cleared:\n%s", out)
}
}

View file

@ -626,6 +626,7 @@ func (c *cmd) queueLs(args []string) int {
var q struct {
Pending int `json:"pending"`
Retrying int `json:"retrying"`
Failed int `json:"failed"`
Oldest *time.Time `json:"oldest"`
Items []struct {
ID int64 `json:"id"`
@ -633,6 +634,7 @@ func (c *cmd) queueLs(args []string) int {
Attempts int `json:"attempts"`
LastError string `json:"last_error"`
NextAttempt time.Time `json:"next_attempt"`
Failed bool `json:"failed"`
} `json:"items"`
}
if err := json.Unmarshal(raw, &q); err != nil {
@ -642,15 +644,22 @@ func (c *cmd) queueLs(args []string) int {
if q.Oldest != nil {
oldest = formatDate(*q.Oldest)
}
fmt.Fprintf(c.io.Out, "Pending: %d\nRetrying: %d\nOldest: %s\n", q.Pending, q.Retrying, oldest)
fmt.Fprintf(c.io.Out, "Pending: %d\nRetrying: %d\nFailed: %d\nOldest: %s\n", q.Pending, q.Retrying, q.Failed, oldest)
if len(q.Items) == 0 {
fmt.Fprintln(c.io.Out, "\nThe queue is empty.")
return 0
}
fmt.Fprintln(c.io.Out)
rows := [][]string{{"ID", "RECIPIENT", "ATTEMPTS", "NEXT ATTEMPT", "LAST ERROR"}}
rows := [][]string{{"ID", "RECIPIENT", "STATE", "ATTEMPTS", "NEXT ATTEMPT", "LAST ERROR"}}
for _, i := range q.Items {
rows = append(rows, []string{strconv.FormatInt(i.ID, 10), i.Recipient, strconv.Itoa(i.Attempts), formatDate(i.NextAttempt), i.LastError})
state, next := "waiting", formatDate(i.NextAttempt)
switch {
case i.Failed:
state, next = "failed", "-" // never retried
case i.Attempts > 0:
state = "retrying"
}
rows = append(rows, []string{strconv.FormatInt(i.ID, 10), i.Recipient, state, strconv.Itoa(i.Attempts), next, i.LastError})
}
c.table(rows)
if q.Pending > len(q.Items) {