feat: show failed deliveries in the API, CLI and UI
This commit is contained in:
parent
bb0858ef87
commit
afd067688c
9 changed files with 109 additions and 14 deletions
|
|
@ -63,11 +63,14 @@ export interface QueueItem {
|
|||
last_error: string
|
||||
next_attempt: string
|
||||
created_at: string
|
||||
/** Given up for good: never retried, kept for a week so the admin can see it. */
|
||||
failed: boolean
|
||||
}
|
||||
|
||||
export interface Queue {
|
||||
pending: number
|
||||
retrying: number
|
||||
failed: number
|
||||
oldest: string | null
|
||||
items: QueueItem[]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ const item = (id: number, over: Partial<QueueItem> = {}): QueueItem => ({
|
|||
last_error: '',
|
||||
next_attempt: '2030-01-02T03:04:05Z',
|
||||
created_at: '2030-01-01T00:00:00Z',
|
||||
failed: false,
|
||||
...over,
|
||||
})
|
||||
|
||||
|
|
@ -51,7 +52,7 @@ describe('queue page', () => {
|
|||
expect(within(fresh).getByText('—')).toBeInTheDocument()
|
||||
|
||||
expect(screen.getByText('Pending').nextElementSibling).toHaveTextContent('2')
|
||||
expect(screen.getByText('Retrying').nextElementSibling).toHaveTextContent('1')
|
||||
expect(screen.getByText('Retrying', { selector: 'div' }).nextElementSibling).toHaveTextContent('1')
|
||||
expect(screen.getByText('Oldest').nextElementSibling).toHaveTextContent('2030-01-01 00:00 UTC')
|
||||
})
|
||||
|
||||
|
|
@ -68,6 +69,26 @@ describe('queue page', () => {
|
|||
expect(screen.queryByText(/showing/i)).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('shows failed deliveries apart from pending ones, with the reason', async () => {
|
||||
await setup([
|
||||
item(1),
|
||||
item(2, { attempts: 2, last_error: 'connection refused' }),
|
||||
item(3, { attempts: 1, failed: true, last_error: 'refused by the relay: 554 policy rejection' }),
|
||||
])
|
||||
const failed = (await screen.findByText('user3@example.com')).closest('tr')!
|
||||
expect(within(failed).getByText('Failed')).toBeInTheDocument()
|
||||
expect(within(failed).getByText(/refused by the relay: 554 policy rejection/)).toBeInTheDocument()
|
||||
expect(within(failed).getByText('—', { selector: 'td' })).toBeInTheDocument() // never retried: no next attempt
|
||||
expect(within(screen.getByText('user2@example.com').closest('tr')!).getByText('Retrying')).toBeInTheDocument()
|
||||
expect(within(screen.getByText('user1@example.com').closest('tr')!).getByText('Waiting')).toBeInTheDocument()
|
||||
|
||||
expect(screen.getByText('Pending').nextElementSibling).toHaveTextContent('2')
|
||||
expect(screen.getByText('Retrying', { selector: 'div' }).nextElementSibling).toHaveTextContent('1')
|
||||
expect(screen.getByText('Failed', { selector: 'div' }).nextElementSibling).toHaveTextContent('1')
|
||||
const rows = screen.getAllByRole('row').slice(1).map((r) => r.textContent ?? '')
|
||||
expect(rows[0]).toContain('user3@example.com') // failed first
|
||||
})
|
||||
|
||||
it('refreshes on demand', async () => {
|
||||
const { user, state } = await setup([item(1)])
|
||||
await screen.findByText('user1@example.com')
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import {
|
|||
AlertDialogHeader,
|
||||
AlertDialogTitle,
|
||||
} from '@/components/ui/alert-dialog'
|
||||
import { Badge } from '@/components/ui/badge'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { Card, CardContent } from '@/components/ui/card'
|
||||
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table'
|
||||
|
|
@ -38,9 +39,10 @@ export function QueuePage() {
|
|||
|
||||
{q && (
|
||||
<>
|
||||
<div className="grid gap-4 sm:grid-cols-3">
|
||||
<div className="grid gap-4 sm:grid-cols-4">
|
||||
<Stat label="Pending" value={String(q.pending)} />
|
||||
<Stat label="Retrying" value={String(q.retrying)} />
|
||||
<Stat label="Failed" value={String(q.failed)} />
|
||||
<Stat label="Oldest" value={q.oldest ? formatDate(q.oldest) : '—'} />
|
||||
</div>
|
||||
{q.items.length === 0 ? (
|
||||
|
|
@ -50,6 +52,7 @@ export function QueuePage() {
|
|||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead>Recipient</TableHead>
|
||||
<TableHead>State</TableHead>
|
||||
<TableHead>Attempts</TableHead>
|
||||
<TableHead>Last error</TableHead>
|
||||
<TableHead>Next attempt</TableHead>
|
||||
|
|
@ -60,9 +63,18 @@ export function QueuePage() {
|
|||
{q.items.map((i) => (
|
||||
<TableRow key={i.id}>
|
||||
<TableCell className="font-medium">{i.recipient}</TableCell>
|
||||
<TableCell>
|
||||
{i.failed ? (
|
||||
<Badge variant="destructive">Failed</Badge>
|
||||
) : i.attempts > 0 ? (
|
||||
<Badge variant="secondary">Retrying</Badge>
|
||||
) : (
|
||||
<Badge variant="outline">Waiting</Badge>
|
||||
)}
|
||||
</TableCell>
|
||||
<TableCell>{i.attempts}</TableCell>
|
||||
<TableCell>{i.last_error || '—'}</TableCell>
|
||||
<TableCell>{formatDate(i.next_attempt)}</TableCell>
|
||||
<TableCell className="max-w-xs whitespace-normal">{i.last_error || '—'}</TableCell>
|
||||
<TableCell>{i.failed ? '—' : formatDate(i.next_attempt)}</TableCell>
|
||||
<TableCell className="text-right">
|
||||
<Button size="sm" variant="outline" onClick={() => setDiscarding(i)}>
|
||||
Discard
|
||||
|
|
|
|||
|
|
@ -71,10 +71,12 @@ export function installFakeApi(state: FakeState) {
|
|||
let m: RegExpMatchArray | null
|
||||
|
||||
if (path === '/queue' && method === 'GET') {
|
||||
const items = [...state.queue].sort((a, b) => b.attempts - a.attempts).slice(0, state.queueLimit)
|
||||
const retrying = state.queue.filter((i) => i.attempts > 0).length
|
||||
const oldest = state.queue.map((i) => i.created_at).sort()[0] ?? null
|
||||
return json(200, { pending: state.queue.length, retrying, oldest, items })
|
||||
const items = [...state.queue].sort((a, b) => Number(b.failed) - Number(a.failed) || b.attempts - a.attempts).slice(0, state.queueLimit)
|
||||
const waiting = state.queue.filter((i) => !i.failed)
|
||||
const retrying = waiting.filter((i) => i.attempts > 0).length
|
||||
const failed = state.queue.length - waiting.length
|
||||
const oldest = waiting.map((i) => i.created_at).sort()[0] ?? null
|
||||
return json(200, { pending: waiting.length, retrying, failed, oldest, items })
|
||||
}
|
||||
if ((m = path.match(/^\/queue\/(\d+)$/)) && method === 'DELETE') {
|
||||
const id = Number(m[1])
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@ components:
|
|||
last_error: {type: string}
|
||||
next_attempt: {type: string, format: date-time}
|
||||
created_at: {type: string, format: date-time}
|
||||
failed: {type: boolean, description: Given up for good (refused by the relay, or too old); never retried, listed for a week}
|
||||
Member:
|
||||
type: object
|
||||
properties:
|
||||
|
|
@ -264,7 +265,8 @@ paths:
|
|||
type: object
|
||||
properties:
|
||||
pending: {type: integer}
|
||||
retrying: {type: integer, description: Deliveries that failed at least once}
|
||||
retrying: {type: integer, description: Waiting deliveries that failed at least once}
|
||||
failed: {type: integer, description: Deliveries given up for good, kept for a week}
|
||||
oldest: {type: string, format: date-time, nullable: true}
|
||||
items: {type: array, items: {$ref: '#/components/schemas/QueueItem'}}
|
||||
/queue/{id}:
|
||||
|
|
|
|||
|
|
@ -523,6 +523,7 @@ type queuedDTO struct {
|
|||
LastError string `json:"last_error"`
|
||||
NextAttempt time.Time `json:"next_attempt"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Failed bool `json:"failed"` // given up for good: never retried
|
||||
}
|
||||
|
||||
func (s *server) getQueue(w http.ResponseWriter, _ *http.Request) {
|
||||
|
|
@ -539,14 +540,14 @@ func (s *server) getQueue(w http.ResponseWriter, _ *http.Request) {
|
|||
out := make([]queuedDTO, 0, len(items))
|
||||
for _, it := range items {
|
||||
out = append(out, queuedDTO{ID: it.ID, Recipient: it.Rcpt, Attempts: it.Attempts, LastError: it.LastError,
|
||||
NextAttempt: it.NextAt, CreatedAt: it.CreatedAt})
|
||||
NextAttempt: it.NextAt, CreatedAt: it.CreatedAt, Failed: it.Failed})
|
||||
}
|
||||
var oldest *time.Time
|
||||
if !st.Oldest.IsZero() {
|
||||
oldest = &st.Oldest
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"pending": st.Pending, "retrying": st.Retrying, "oldest": oldest, "items": out,
|
||||
"pending": st.Pending, "retrying": st.Retrying, "failed": st.Failed, "oldest": oldest, "items": out,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import (
|
|||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
|
@ -448,6 +449,7 @@ func queueServer(t *testing.T) (http.Handler, *store.Store) {
|
|||
type queueJSON struct {
|
||||
Pending int `json:"pending"`
|
||||
Retrying int `json:"retrying"`
|
||||
Failed int `json:"failed"`
|
||||
Oldest *time.Time `json:"oldest"`
|
||||
Items []struct {
|
||||
ID int64 `json:"id"`
|
||||
|
|
@ -456,6 +458,7 @@ type queueJSON struct {
|
|||
LastError string `json:"last_error"`
|
||||
NextAttempt time.Time `json:"next_attempt"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Failed bool `json:"failed"`
|
||||
} `json:"items"`
|
||||
}
|
||||
|
||||
|
|
@ -630,3 +633,24 @@ func TestArchiveImportStoppedByAReadErrorIsAnErrorResponse(t *testing.T) {
|
|||
t.Fatalf("the first message was complete and is kept, got %d", total)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueueShowsFailedDeliveriesApartFromPendingOnes(t *testing.T) {
|
||||
h, s := queueServer(t)
|
||||
s.Enqueue([]byte("x"), []store.NewOutbound{{From: "b@x.org", Rcpt: "a@x.com"}, {From: "b@x.org", Rcpt: "b@x.com"}, {From: "b@x.org", Rcpt: "c@x.com"}})
|
||||
due, _ := s.DueOutbox(10)
|
||||
s.FailOutbox(due[2].ID, "refused by the relay: 554 no")
|
||||
s.RetryOutbox(due[1].ID, time.Now().Add(time.Hour), "down")
|
||||
|
||||
var q queueJSON
|
||||
decode(t, do(t, h, "GET", "/api/v1/queue", "", true), &q)
|
||||
if q.Pending != 2 || q.Retrying != 1 || q.Failed != 1 || len(q.Items) != 3 {
|
||||
t.Fatalf("queue: %+v", q)
|
||||
}
|
||||
if !q.Items[0].Failed || q.Items[0].Recipient != "c@x.com" || q.Items[0].LastError != "refused by the relay: 554 no" {
|
||||
t.Fatalf("failed deliveries come first, with the reason: %+v", q.Items[0])
|
||||
}
|
||||
if q.Items[1].Failed || q.Items[2].Failed {
|
||||
t.Fatalf("only one failed: %+v", q.Items)
|
||||
}
|
||||
expect(t, do(t, h, "DELETE", "/api/v1/queue/"+strconv.FormatInt(due[2].ID, 10), "", true), 204)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -776,3 +776,24 @@ func TestDKIMDomainIsLowercased(t *testing.T) {
|
|||
t.Fatalf("lookup name: %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueueShowsFailedDeliveries(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
e.s.Enqueue([]byte("x"), []store.NewOutbound{{From: "b@x.org", Rcpt: "waits@x.com"}, {From: "b@x.org", Rcpt: "retries@x.com"}, {From: "b@x.org", Rcpt: "lost@x.com"}})
|
||||
due, _ := e.s.DueOutbox(10)
|
||||
e.s.RetryOutbox(due[1].ID, time.Now().Add(time.Hour), "connection refused")
|
||||
e.s.FailOutbox(due[2].ID, "refused by the relay: 554 policy rejection")
|
||||
out := e.ok(t, "queue", "ls")
|
||||
for _, want := range []string{"Pending: 2", "Retrying: 1", "Failed: 1", "STATE", "failed", "retrying", "waiting", "refused by the relay: 554 policy rejection"} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Fatalf("queue ls lacks %q:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
if strings.Index(out, "lost@x.com") > strings.Index(out, "retries@x.com") || strings.Index(out, "retries@x.com") > strings.Index(out, "waits@x.com") {
|
||||
t.Fatalf("failed first, then retrying, then waiting:\n%s", out)
|
||||
}
|
||||
e.ok(t, "queue", "rm", fmt.Sprint(due[2].ID))
|
||||
if out := e.ok(t, "queue", "ls"); strings.Contains(out, "lost@x.com") || !strings.Contains(out, "Failed: 0") {
|
||||
t.Fatalf("a failed delivery can be cleared:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -626,6 +626,7 @@ func (c *cmd) queueLs(args []string) int {
|
|||
var q struct {
|
||||
Pending int `json:"pending"`
|
||||
Retrying int `json:"retrying"`
|
||||
Failed int `json:"failed"`
|
||||
Oldest *time.Time `json:"oldest"`
|
||||
Items []struct {
|
||||
ID int64 `json:"id"`
|
||||
|
|
@ -633,6 +634,7 @@ func (c *cmd) queueLs(args []string) int {
|
|||
Attempts int `json:"attempts"`
|
||||
LastError string `json:"last_error"`
|
||||
NextAttempt time.Time `json:"next_attempt"`
|
||||
Failed bool `json:"failed"`
|
||||
} `json:"items"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &q); err != nil {
|
||||
|
|
@ -642,15 +644,22 @@ func (c *cmd) queueLs(args []string) int {
|
|||
if q.Oldest != nil {
|
||||
oldest = formatDate(*q.Oldest)
|
||||
}
|
||||
fmt.Fprintf(c.io.Out, "Pending: %d\nRetrying: %d\nOldest: %s\n", q.Pending, q.Retrying, oldest)
|
||||
fmt.Fprintf(c.io.Out, "Pending: %d\nRetrying: %d\nFailed: %d\nOldest: %s\n", q.Pending, q.Retrying, q.Failed, oldest)
|
||||
if len(q.Items) == 0 {
|
||||
fmt.Fprintln(c.io.Out, "\nThe queue is empty.")
|
||||
return 0
|
||||
}
|
||||
fmt.Fprintln(c.io.Out)
|
||||
rows := [][]string{{"ID", "RECIPIENT", "ATTEMPTS", "NEXT ATTEMPT", "LAST ERROR"}}
|
||||
rows := [][]string{{"ID", "RECIPIENT", "STATE", "ATTEMPTS", "NEXT ATTEMPT", "LAST ERROR"}}
|
||||
for _, i := range q.Items {
|
||||
rows = append(rows, []string{strconv.FormatInt(i.ID, 10), i.Recipient, strconv.Itoa(i.Attempts), formatDate(i.NextAttempt), i.LastError})
|
||||
state, next := "waiting", formatDate(i.NextAttempt)
|
||||
switch {
|
||||
case i.Failed:
|
||||
state, next = "failed", "-" // never retried
|
||||
case i.Attempts > 0:
|
||||
state = "retrying"
|
||||
}
|
||||
rows = append(rows, []string{strconv.FormatInt(i.ID, 10), i.Recipient, state, strconv.Itoa(i.Attempts), next, i.LastError})
|
||||
}
|
||||
c.table(rows)
|
||||
if q.Pending > len(q.Items) {
|
||||
|
|
|
|||
Loading…
Reference in a new issue