Merge pull request 'Fix the two High findings of the global review' (#31) from fix/review-high into main

This commit is contained in:
squidd 2026-11-10 11:44:02 +01:00
commit b09ee2e419
20 changed files with 827 additions and 81 deletions

View file

@ -179,8 +179,11 @@ It answers `451` only when the database itself fails.
- A permanent rejection of the message by the relay (5xx: policy, size, line length...) is never retried, but the
delivery is not made to vanish: it is kept as **failed**, with the reason, for a week (queue API, `parsimony queue ls`,
the Queue page; discard it to clear it). The same happens when a delivery is still failing after 3 days.
- A bad-mailbox reply (`5.1.x`, the address does not exist) drops the delivery and counts as a hard bounce for the
member (3 disable it). Other errors, such as "relay access denied", never disable members.
- A reply to `RCPT TO` that says the address does not exist (`5.1.1`, `5.1.2`, `5.1.3`, `5.1.6`, `5.1.10`) drops the
delivery and counts as a hard bounce for the member (3 disable it). Nothing else counts: not "relay access denied", not a
refused sender (`5.1.7`, `5.1.8`, or any code answering `MAIL FROM`), not a refusal after `DATA`. Those deliveries are kept
as failed.
- A relay that accepted the message is never asked again because `QUIT` went wrong.
- Delivery is at-least-once: a crash between the relay accepting a message and the queue entry being
removed can send one message to one recipient twice.
- `GET /api/v1/queue` and the Queue page show what is pending or failing, and a stuck delivery can be discarded.
@ -246,6 +249,10 @@ lists.example.org lmtp:inet:127.0.0.1:2424
and `Reply-To` come from the header; so a post whose header names someone else (or has no usable `From`, or several
addresses) is held for moderation instead of delivered, with the reason shown to the admin (API, CLI, UI). Let Postfix
reject forged senders too (SPF, DKIM, DMARC checks on inbound mail).
- **The moderation queue is bounded.** A list holds at most 500 messages and 200 MiB, and one held message at most 10 MiB; a
full queue answers `452` (the sender retries later), a message over 10 MiB `552`. Messages nobody moderated for 30 days
are dropped. Listings (API, CLI, UI) show a summary and never read the bodies. The held view shows the From header
readers would see next to the real sender and flags a difference.
- **API token.** `parsimony serve` refuses a token shorter than 24 characters or containing whitespace
(`openssl rand -hex 24` makes a good one). After 10 wrong or missing tokens within 5 minutes an address is
locked out for 5 minutes (HTTP 429, valid token included). The address is the TCP peer: behind a reverse proxy

View file

@ -30,6 +30,10 @@ export interface Held {
created_at: string
/** Why the message was held, for the admin. */
reason: string
/** The From header as received: the author readers will see. */
from: string
/** The From header names someone other than the envelope sender, or nobody usable. */
mismatch: boolean
}
export interface Message {

View file

@ -50,7 +50,7 @@ describe('list page', () => {
installFakeApi(
newState({
lists: [dev()],
held: { 1: [{ id: 5, sender: 'a@x.com', subject: 's', size: 10, created_at: '2030-01-01T00:00:00Z', reason: '' }] },
held: { 1: [{ id: 5, sender: 'a@x.com', subject: 's', size: 10, created_at: '2030-01-01T00:00:00Z', reason: '', from: 'a@x.com', mismatch: false }] },
}),
)
renderApp('/lists/1', { token: 'good' })

View file

@ -11,6 +11,8 @@ const held = (id: number, over: Partial<Held> = {}): Held => ({
size: 2048,
created_at: '2030-01-02T03:04:05Z',
reason: 'the list is moderated',
from: `Sender ${id} <s${id}@example.com>`,
mismatch: false,
...over,
})
@ -46,6 +48,29 @@ describe('held tab', () => {
expect(screen.getByRole('columnheader', { name: /reason/i })).toBeInTheDocument()
})
it('shows the author readers will see, and flags a From header that is not the sender', async () => {
await setup([
held(1),
held(2, { sender: 'bob@example.com', from: 'Eve <eve@example.com>', mismatch: true }),
held(3, { sender: 'bob@example.com', from: '', mismatch: true }),
])
const honest = (await screen.findByText('Subject 1')).closest('tr')!
expect(within(honest).getByText('Sender 1 <s1@example.com>')).toBeInTheDocument()
expect(within(honest).getByText('s1@example.com')).toBeInTheDocument()
expect(within(honest).queryByText(/differs from the sender/i)).not.toBeInTheDocument()
const passing = screen.getByText('Subject 2').closest('tr')!
expect(within(passing).getByText('Eve <eve@example.com>')).toBeInTheDocument()
expect(within(passing).getByText('bob@example.com')).toBeInTheDocument() // who really sent it
expect(within(passing).getByText(/differs from the sender/i)).toBeInTheDocument()
const noFrom = screen.getByText('Subject 3').closest('tr')!
expect(within(noFrom).getByText(/differs from the sender/i)).toBeInTheDocument()
expect(within(noFrom).getByText('(none)')).toBeInTheDocument()
expect(screen.getByRole('columnheader', { name: /^from \(header\)$/i })).toBeInTheDocument()
expect(screen.getByRole('columnheader', { name: /^sender$/i })).toBeInTheDocument()
})
it('says nothing is waiting', async () => {
await setup([])
expect(await screen.findByText(/nothing is waiting/i)).toBeInTheDocument()

View file

@ -11,6 +11,7 @@ import {
AlertDialogHeader,
AlertDialogTitle,
} from '@/components/ui/alert-dialog'
import { Badge } from '@/components/ui/badge'
import { Button } from '@/components/ui/button'
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table'
import { api, type Held, type List } from '@/lib/api'
@ -60,7 +61,8 @@ export function HeldTab() {
<TableHeader>
<TableRow>
<TableHead>Subject</TableHead>
<TableHead>From</TableHead>
<TableHead>From (header)</TableHead>
<TableHead>Sender</TableHead>
<TableHead>Received</TableHead>
<TableHead>Size</TableHead>
<TableHead>Reason</TableHead>
@ -71,6 +73,14 @@ export function HeldTab() {
{items.map((h) => (
<TableRow key={h.id}>
<TableCell className="font-medium">{h.subject || '(no subject)'}</TableCell>
<TableCell className="whitespace-normal">
{h.from || <span className="text-muted-foreground">(none)</span>}
{h.mismatch && (
<Badge variant="destructive" className="ml-2">
Differs from the sender
</Badge>
)}
</TableCell>
<TableCell>{h.sender}</TableCell>
<TableCell>{formatDate(h.created_at)}</TableCell>
<TableCell>{formatBytes(h.size)}</TableCell>

View file

@ -56,9 +56,12 @@ components:
type: object
properties:
id: {type: integer}
sender: {type: string}
subject: {type: string}
sender: {type: string, description: The SMTP (envelope) sender}
from: {type: string, description: "The From header as received: the author readers will see (at most 200 characters)"}
mismatch: {type: boolean, description: The From header names someone other than the sender, or no usable address}
subject: {type: string, description: Decoded, at most 200 characters}
size: {type: integer}
reason: {type: string, description: Why the message was held}
created_at: {type: string, format: date-time}
Error:
type: object

View file

@ -2,13 +2,11 @@
package api
import (
"bytes"
"context"
"crypto/subtle"
"encoding/json"
"errors"
"fmt"
"mime"
"net/http"
"net/mail"
"strconv"
@ -353,18 +351,17 @@ type heldDTO struct {
Size int `json:"size"`
CreatedAt time.Time `json:"created_at"`
Reason string `json:"reason"` // why the message was held
// The From header as received, and whether it names someone other than the envelope sender:
// what recipients will see as the author, against who really sent it.
From string `json:"from"`
Mismatch bool `json:"mismatch"`
}
func subjectOf(data []byte) string {
m, err := mail.ReadMessage(bytes.NewReader(data))
if err != nil {
return ""
}
subj := m.Header.Get("Subject")
if dec, err := new(mime.WordDecoder).DecodeHeader(subj); err == nil {
return dec
}
return subj
// fromMismatch reports whether the From header names an address other than the envelope sender, or
// no usable address at all.
func fromMismatch(fromHeader, sender string) bool {
a, err := mail.ParseAddress(fromHeader)
return err != nil || !strings.EqualFold(a.Address, sender)
}
func (s *server) listHeld(w http.ResponseWriter, r *http.Request) {
@ -383,7 +380,8 @@ func (s *server) listHeld(w http.ResponseWriter, r *http.Request) {
}
out := make([]heldDTO, 0, len(hs))
for _, h := range hs {
out = append(out, heldDTO{ID: h.ID, Sender: h.Sender, Subject: subjectOf(h.Data), Size: len(h.Data), CreatedAt: h.CreatedAt, Reason: h.Reason})
out = append(out, heldDTO{ID: h.ID, Sender: h.Sender, Subject: h.Subject, Size: h.Size, CreatedAt: h.CreatedAt, Reason: h.Reason,
From: h.FromHeader, Mismatch: fromMismatch(h.FromHeader, h.Sender)})
}
writeJSON(w, http.StatusOK, out)
}

View file

@ -271,11 +271,13 @@ func heldServer(t *testing.T) (http.Handler, *store.Store, *fakeReleaser) {
}
type heldJSON struct {
ID int64 `json:"id"`
Sender string `json:"sender"`
Subject string `json:"subject"`
Size int `json:"size"`
Reason string `json:"reason"`
ID int64 `json:"id"`
Sender string `json:"sender"`
From string `json:"from"`
Mismatch bool `json:"mismatch"`
Subject string `json:"subject"`
Size int `json:"size"`
Reason string `json:"reason"`
}
func TestHeldQueueAPI(t *testing.T) {
@ -298,6 +300,35 @@ func TestHeldQueueAPI(t *testing.T) {
expect(t, do(t, h, "DELETE", "/api/v1/held/abc", "", true), 404)
}
func TestHeldListingShowsWhoReallyWrote(t *testing.T) {
s, _ := store.Open(":memory:")
defer s.Close()
l, _ := s.CreateList("dev", "example.org", "Dev")
s.HoldMessage(l.ID, "bob@x.com", []byte("From: Bob <BOB@x.com>\r\nSubject: honest\r\n\r\nb"), "")
s.HoldMessage(l.ID, "bob@x.com", []byte("From: Eve <eve@x.com>\r\nSubject: passing as Eve\r\n\r\nb"), "")
s.HoldMessage(l.ID, "bob@x.com", []byte("Subject: no From at all\r\n\r\nb"), "")
s.HoldMessage(l.ID, "bob@x.com", []byte("From: not an address\r\nSubject: garbled\r\n\r\nb"), "")
h := New(s, token, nil)
var hs []heldJSON
decode(t, do(t, h, "GET", "/api/v1/lists/1/held", "", true), &hs)
if len(hs) != 4 {
t.Fatalf("%+v", hs)
}
want := []struct {
from string
mismatch bool
}{{"Bob <BOB@x.com>", false}, {"Eve <eve@x.com>", true}, {"", true}, {"not an address", true}}
for i, w := range want {
if hs[i].From != w.from || hs[i].Mismatch != w.mismatch {
t.Errorf("held %d: from %q mismatch %v, want %q %v", i+1, hs[i].From, hs[i].Mismatch, w.from, w.mismatch)
}
if hs[i].Sender != "bob@x.com" {
t.Errorf("the envelope sender stays visible: %q", hs[i].Sender)
}
}
}
func TestHeldRequiresAuth(t *testing.T) {
h, _, _ := heldServer(t)
expect(t, do(t, h, "POST", "/api/v1/held/1/approve", "", false), 401)

View file

@ -319,7 +319,7 @@ func TestHeldQueue(t *testing.T) {
id2, _ := e.s.HoldMessage(l.ID, "eve@x.com", []byte("Subject: Second held\r\n\r\nbody"), "")
id3, _ := e.s.HoldMessage(l.ID, "anon@x.com", []byte("From: anon@x.com\r\n\r\nno subject here"), "")
out := e.ok(t, "held", "ls", "dev@example.org")
for _, want := range []string{"ID", "FROM", "SUBJECT", "SIZE", "REASON", "bob@x.com", "First held", "eve@x.com", "Second held", "(no subject)", "the list is moderated"} {
for _, want := range []string{"ID", "SENDER", "FROM", "SUBJECT", "SIZE", "REASON", "bob@x.com", "First held", "eve@x.com", "Second held", "(no subject)", "the list is moderated"} {
if !strings.Contains(out, want) {
t.Fatalf("held ls lacks %q:\n%s", want, out)
}
@ -797,3 +797,30 @@ func TestQueueShowsFailedDeliveries(t *testing.T) {
t.Fatalf("a failed delivery can be cleared:\n%s", out)
}
}
func TestHeldListingWarnsWhenTheHeaderDiffersFromTheSender(t *testing.T) {
e := newEnv(t)
e.ok(t, "list", "add", "dev@example.org")
l, _ := e.s.FindList("dev@example.org")
e.s.HoldMessage(l.ID, "bob@x.com", []byte("From: Bob <bob@x.com>\r\nSubject: honest\r\n\r\nb"), "r")
e.s.HoldMessage(l.ID, "bob@x.com", []byte("From: Eve <eve@x.com>\r\nSubject: passing as eve\r\n\r\nb"), "r")
e.s.HoldMessage(l.ID, "bob@x.com", []byte("Subject: no from\r\n\r\nb"), "r")
out := e.ok(t, "held", "ls", "dev@example.org")
lines := map[string]string{}
for _, l := range strings.Split(out, "\n") {
for _, key := range []string{"honest", "passing as eve", "no from"} {
if strings.Contains(l, key) {
lines[key] = l
}
}
}
if strings.Contains(lines["honest"], "differs") {
t.Fatalf("a matching header is not flagged: %q", lines["honest"])
}
if !strings.Contains(lines["passing as eve"], "Eve <eve@x.com>") || !strings.Contains(lines["passing as eve"], "differs") {
t.Fatalf("the author readers will see is shown and flagged: %q", lines["passing as eve"])
}
if !strings.Contains(lines["no from"], "differs") {
t.Fatalf("no usable From is flagged too: %q", lines["no from"])
}
}

View file

@ -375,6 +375,8 @@ func (c *cmd) heldLs(args []string) int {
Size int `json:"size"`
CreatedAt time.Time `json:"created_at"`
Reason string `json:"reason"`
From string `json:"from"`
Mismatch bool `json:"mismatch"`
}
if err := json.Unmarshal(raw, &hs); err != nil {
return c.fail(err)
@ -383,13 +385,20 @@ func (c *cmd) heldLs(args []string) int {
fmt.Fprintln(c.io.Out, "Nothing is held for moderation.")
return 0
}
rows := [][]string{{"ID", "RECEIVED", "FROM", "SUBJECT", "SIZE", "REASON"}}
rows := [][]string{{"ID", "RECEIVED", "SENDER", "FROM", "SUBJECT", "SIZE", "REASON"}}
for _, h := range hs {
subject := h.Subject
if subject == "" {
subject = "(no subject)"
}
rows = append(rows, []string{strconv.FormatInt(h.ID, 10), formatDate(h.CreatedAt), h.Sender, subject, formatBytes(h.Size), h.Reason})
from := h.From
if from == "" {
from = "-"
}
if h.Mismatch {
from += " (differs)" // readers would see someone other than the sender
}
rows = append(rows, []string{strconv.FormatInt(h.ID, 10), formatDate(h.CreatedAt), h.Sender, from, subject, formatBytes(h.Size), h.Reason})
}
c.table(rows)
return 0

View file

@ -92,6 +92,8 @@ func (s *session) status(rcpt string, err error) error {
return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such list"}
case errors.Is(err, router.ErrHeldFull):
return &smtp.SMTPError{Code: 452, EnhancedCode: smtp.EnhancedCode{4, 2, 2}, Message: "the moderation queue of this list is full, try again later"}
case errors.Is(err, router.ErrHeldTooLarge):
return &smtp.SMTPError{Code: 552, EnhancedCode: smtp.EnhancedCode{5, 3, 4}, Message: "message too large for this moderated list"}
case errors.Is(err, router.ErrNotMember):
return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "only list members may post"}
case errors.Is(err, router.ErrBadToken):

View file

@ -289,3 +289,30 @@ func TestAFullModerationQueueAnswers452(t *testing.T) {
t.Fatalf("a temporary refusal, so a legitimate sender retries later and a spam run goes away: %v", res["dev@example.org"])
}
}
func TestAMessageTooBigToHoldAnswers552(t *testing.T) {
st := newStack(t, nil)
l, _ := st.store.FindList("dev@example.org")
st.store.SetPolicies(l.ID, store.PostModerated, store.JoinOpen, store.ArchivePrivate)
conn, err := net.Dial("tcp", st.addr)
if err != nil {
t.Fatal(err)
}
c := smtp.NewClientLMTP(conn)
defer c.Close()
c.Hello("localhost")
c.Mail("alice@example.com", nil)
c.Rcpt("dev@example.org", nil)
w, _ := c.Data()
w.Write([]byte("From: Alice <alice@example.com>\r\nSubject: huge\r\n\r\n"))
line := []byte(strings.Repeat("x", 76) + "\r\n")
for i := 0; i < (10<<20)/len(line)+10; i++ {
w.Write(line)
}
_, err = w.CloseWithLMTPResponse()
var de smtp.LMTPDataError
var se *smtp.SMTPError
if !errors.As(err, &de) || !errors.As(de["dev@example.org"], &se) || se.Code != 552 {
t.Fatalf("a permanent refusal: the sender is told to send it elsewhere: %v", err)
}
}

View file

@ -8,6 +8,7 @@ import (
"crypto/x509"
"errors"
"fmt"
"io"
"net"
"strings"
"sync"
@ -176,10 +177,50 @@ func (r *SMTPRelay) sendBatch(ctx context.Context, from string, to []string, dat
}
}()
if err := c.SendMail(from, to, bytes.NewReader(data)); err != nil {
return authError(err)
// The commands are issued one by one, rather than with SendMail, to know which one failed.
var mailOpts *smtp.MailOptions
if !isASCII(from) || !allASCII(to) {
mailOpts = &smtp.MailOptions{UTF8: true}
}
return c.Quit()
if err := c.Mail(from, mailOpts); err != nil {
return authError(&PhaseError{Phase: "MAIL", Err: err})
}
for _, rcpt := range to {
if err := c.Rcpt(rcpt, nil); err != nil {
return authError(&PhaseError{Phase: "RCPT", Err: err})
}
}
w, err := c.Data()
if err != nil {
return authError(&PhaseError{Phase: "DATA", Err: err})
}
if _, err := io.Copy(w, bytes.NewReader(data)); err != nil {
return &PhaseError{Phase: "DATA", Err: err}
}
if err := w.Close(); err != nil {
return &PhaseError{Phase: "DATA", Err: err}
}
// The relay has accepted the message. If QUIT now fails, retrying would deliver it twice.
c.Quit()
return nil
}
func isASCII(s string) bool {
for i := 0; i < len(s); i++ {
if s[i] >= 0x80 {
return false
}
}
return true
}
func allASCII(ss []string) bool {
for _, s := range ss {
if !isASCII(s) {
return false
}
}
return true
}
// Sent is a message recorded by Fake.
@ -230,15 +271,52 @@ func IsPermanent(err error) bool {
return errors.As(err, &se) && se.Code >= 500 && se.Code <= 599
}
// IsBadMailbox reports whether err says the destination mailbox is invalid (SMTP
// enhanced status 5.1.x). Other permanent errors, such as "relay access denied",
// say nothing about the recipient.
// PhaseError says which SMTP command a failure answers. The same enhanced status means different
// things depending on it: 5.1.1 to RCPT is "this mailbox does not exist", but to MAIL FROM it is
// about our own sender address and says nothing about the recipient.
type PhaseError struct {
Phase string // "MAIL", "RCPT" or "DATA"
Err error
}
// Error names the command, so the admin can tell a refused recipient from a refused sender.
func (e *PhaseError) Error() string {
label := map[string]string{"MAIL": "MAIL FROM", "RCPT": "RCPT TO", "DATA": "DATA"}[e.Phase]
if label == "" {
label = e.Phase
}
return label + ": " + e.Err.Error()
}
func (e *PhaseError) Unwrap() error { return e.Err }
// AtRcpt marks err as the relay's answer to RCPT TO (SMTPRelay does this itself; it is exported
// for tests that fake a relay).
func AtRcpt(err error) error {
if err == nil {
return nil
}
return &PhaseError{Phase: "RCPT", Err: err}
}
// recipientUnknown are the enhanced status codes that say the destination mailbox is invalid:
// 5.1.1 bad mailbox, 5.1.2 bad domain, 5.1.3 bad address syntax, 5.1.6 moved, 5.1.10 null MX.
var recipientUnknown = map[smtp.EnhancedCode]bool{
{5, 1, 1}: true, {5, 1, 2}: true, {5, 1, 3}: true, {5, 1, 6}: true, {5, 1, 10}: true,
}
// IsBadMailbox reports whether err is the relay saying the recipient does not exist: a 5.1.1, 5.1.2,
// 5.1.3, 5.1.6 or 5.1.10 answer to RCPT TO. Anything else, such as "relay access denied", a policy
// refusal or a rejected sender (5.1.7, 5.1.8, or any code answering MAIL FROM), says nothing about
// the recipient and must not count against a member.
func IsBadMailbox(err error) bool {
var pe *PermanentError
if errors.As(err, &pe) {
return pe.BadMailbox
}
var ph *PhaseError
if !errors.As(err, &ph) || ph.Phase != "RCPT" {
return false
}
var se *smtp.SMTPError
return errors.As(err, &se) && se.Code >= 500 && se.Code <= 599 &&
se.EnhancedCode[0] == 5 && se.EnhancedCode[1] == 1
return errors.As(err, &se) && se.Code >= 500 && se.Code <= 599 && recipientUnknown[se.EnhancedCode]
}

View file

@ -145,13 +145,25 @@ func TestClassifyErrors(t *testing.T) {
{"plain", errors.New("connection refused"), false, false},
{"451", smtpErr(451, smtp.EnhancedCode{4, 3, 0}), false, false},
{"421", smtpErr(421, smtp.NoEnhancedCode), false, false},
{"550 unknown mailbox", smtpErr(550, smtp.EnhancedCode{5, 1, 1}), true, true},
{"553 bad address syntax", smtpErr(553, smtp.EnhancedCode{5, 1, 3}), true, true},
{"451 at RCPT", AtRcpt(smtpErr(451, smtp.EnhancedCode{4, 1, 1})), false, false},
{"554 relay denied", smtpErr(554, smtp.EnhancedCode{5, 7, 1}), true, false},
{"550 no enhanced code", smtpErr(550, smtp.NoEnhancedCode), true, false},
{"wrapped", fmt.Errorf("batch 0-1: %w", smtpErr(550, smtp.EnhancedCode{5, 1, 1})), true, true},
{"explicit permanent", &PermanentError{Err: errors.New("no")}, true, false},
{"explicit bad mailbox", &PermanentError{Err: errors.New("no"), BadMailbox: true}, true, true},
// The same enhanced code means different things depending on the command it answers.
{"5.1.1 without a command", smtpErr(550, smtp.EnhancedCode{5, 1, 1}), true, false},
{"5.1.1 at RCPT", AtRcpt(smtpErr(550, smtp.EnhancedCode{5, 1, 1})), true, true},
{"5.1.2 at RCPT", AtRcpt(smtpErr(550, smtp.EnhancedCode{5, 1, 2})), true, true},
{"5.1.3 at RCPT", AtRcpt(smtpErr(553, smtp.EnhancedCode{5, 1, 3})), true, true},
{"5.1.6 at RCPT", AtRcpt(smtpErr(550, smtp.EnhancedCode{5, 1, 6})), true, true},
{"5.1.10 at RCPT", AtRcpt(smtpErr(550, smtp.EnhancedCode{5, 1, 10})), true, true},
{"wrapped, at RCPT", AtRcpt(fmt.Errorf("batch 0-1: %w", smtpErr(550, smtp.EnhancedCode{5, 1, 1}))), true, true},
{"5.1.0 at RCPT", AtRcpt(smtpErr(550, smtp.EnhancedCode{5, 1, 0})), true, false},
{"5.1.4 at RCPT", AtRcpt(smtpErr(550, smtp.EnhancedCode{5, 1, 4})), true, false},
{"5.1.7 at RCPT (bad sender)", AtRcpt(smtpErr(553, smtp.EnhancedCode{5, 1, 7})), true, false},
{"5.1.8 at RCPT (bad sender)", AtRcpt(smtpErr(553, smtp.EnhancedCode{5, 1, 8})), true, false},
{"5.2.2 at RCPT (mailbox full)", AtRcpt(smtpErr(552, smtp.EnhancedCode{5, 2, 2})), true, false},
{"5.7.1 at RCPT (policy)", AtRcpt(smtpErr(554, smtp.EnhancedCode{5, 7, 1})), true, false},
}
for _, c := range cases {
if got := IsPermanent(c.err); got != c.perm {
@ -163,28 +175,40 @@ func TestClassifyErrors(t *testing.T) {
}
}
type rejecting struct{ err error }
// rejecting refuses one stage of the SMTP transaction.
type rejecting struct{ mail, rcpt, data error }
func (r rejecting) NewSession(*smtp.Conn) (smtp.Session, error) { return &rejectSession{r.err}, nil }
func (r rejecting) NewSession(*smtp.Conn) (smtp.Session, error) { return &rejectSession{r}, nil }
type rejectSession struct{ err error }
type rejectSession struct{ r rejecting }
func (s *rejectSession) Reset() {}
func (s *rejectSession) Logout() error { return nil }
func (s *rejectSession) Mail(string, *smtp.MailOptions) error { return nil }
func (s *rejectSession) Rcpt(string, *smtp.RcptOptions) error { return s.err }
func (s *rejectSession) Data(io.Reader) error { return nil }
func (s *rejectSession) Mail(string, *smtp.MailOptions) error { return s.r.mail }
func (s *rejectSession) Rcpt(string, *smtp.RcptOptions) error { return s.r.rcpt }
func (s *rejectSession) Data(io.Reader) error { return s.r.data }
func TestRelayRejectionsAreClassifiedFromTheWire(t *testing.T) {
func se(code int, enh smtp.EnhancedCode, msg string) error {
return &smtp.SMTPError{Code: code, EnhancedCode: enh, Message: msg}
}
func TestRelayRejectionsAreClassifiedByTheCommandTheyAnswer(t *testing.T) {
for name, tc := range map[string]struct {
err error
r rejecting
perm, mailbox bool
}{
"unknown user": {&smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"}, true, true},
"relay denied": {&smtp.SMTPError{Code: 554, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "relay access denied"}, true, false},
"try later": {&smtp.SMTPError{Code: 451, EnhancedCode: smtp.EnhancedCode{4, 3, 0}, Message: "try later"}, false, false},
"unknown user at RCPT": {rejecting{rcpt: se(550, smtp.EnhancedCode{5, 1, 1}, "no such user")}, true, true},
"relay denied at RCPT": {rejecting{rcpt: se(554, smtp.EnhancedCode{5, 7, 1}, "relay access denied")}, true, false},
"try later at RCPT": {rejecting{rcpt: se(451, smtp.EnhancedCode{4, 3, 0}, "try later")}, false, false},
"bad sender 5.1.8 at MAIL": {rejecting{mail: se(550, smtp.EnhancedCode{5, 1, 8}, "bad outbound sender")}, true, false},
"sender domain 5.1.0 at MAIL": {rejecting{mail: se(550, smtp.EnhancedCode{5, 1, 0}, "sender domain not found")}, true, false},
"unlisted sender 5.1.7 at MAIL": {rejecting{mail: se(553, smtp.EnhancedCode{5, 1, 7}, "sender address rejected")}, true, false},
"even 5.1.1 at MAIL is the sender": {rejecting{mail: se(550, smtp.EnhancedCode{5, 1, 1}, "sender unknown")}, true, false},
"5.1.1 after DATA is not a mailbox": {rejecting{data: se(550, smtp.EnhancedCode{5, 1, 1}, "content refused")}, true, false},
"policy refusal after DATA": {rejecting{data: se(554, smtp.EnhancedCode{5, 7, 1}, "spam")}, true, false},
"temporary after DATA": {rejecting{data: se(451, smtp.EnhancedCode{4, 3, 0}, "try later")}, false, false},
} {
srv := smtp.NewServer(rejecting{tc.err})
srv := smtp.NewServer(tc.r)
ln, _ := net.Listen("tcp", "127.0.0.1:0")
go srv.Serve(ln)
r := &SMTPRelay{Addr: ln.Addr().String()}
@ -196,6 +220,105 @@ func TestRelayRejectionsAreClassifiedFromTheWire(t *testing.T) {
if IsPermanent(err) != tc.perm || IsBadMailbox(err) != tc.mailbox {
t.Errorf("%s: permanent=%v mailbox=%v for %v", name, IsPermanent(err), IsBadMailbox(err), err)
}
// The error says which command was refused, for the admin reading the queue.
stage := map[bool]string{true: "RCPT TO: ", false: ""}[tc.r.rcpt != nil]
if tc.r.mail != nil {
stage = "MAIL FROM: "
}
if tc.r.data != nil {
stage = "DATA: "
}
if !strings.HasPrefix(err.Error(), stage) {
t.Errorf("%s: error %q should start with %q", name, err.Error(), stage)
}
}
}
// quitDropper is a bare SMTP server that accepts the message and then hangs up on QUIT.
type quitDropper struct {
mu sync.Mutex
accepted int
}
func startQuitDropper(t *testing.T, reply string) (*quitDropper, string) {
t.Helper()
q := &quitDropper{}
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { ln.Close() })
go func() {
for {
c, err := ln.Accept()
if err != nil {
return
}
go func() {
defer c.Close()
c.Write([]byte("220 test ESMTP\r\n"))
buf := make([]byte, 0, 4096)
one := make([]byte, 1)
inData := false
line := func() (string, bool) {
buf = buf[:0]
for {
if _, err := c.Read(one); err != nil {
return "", false
}
buf = append(buf, one[0])
if len(buf) >= 2 && buf[len(buf)-2] == '\r' && buf[len(buf)-1] == '\n' {
return string(buf[:len(buf)-2]), true
}
}
}
for {
l, ok := line()
if !ok {
return
}
switch {
case inData:
if l == "." {
inData = false
q.mu.Lock()
q.accepted++
q.mu.Unlock()
c.Write([]byte("250 2.0.0 queued\r\n"))
}
case strings.HasPrefix(l, "EHLO"), strings.HasPrefix(l, "HELO"):
c.Write([]byte("250 test\r\n"))
case strings.HasPrefix(l, "MAIL"), strings.HasPrefix(l, "RCPT"):
c.Write([]byte("250 2.1.0 ok\r\n"))
case l == "DATA":
inData = true
c.Write([]byte("354 go ahead\r\n"))
case l == "QUIT":
if reply != "" {
c.Write([]byte(reply))
}
return // hang up
}
}
}()
}
}()
return q, ln.Addr().String()
}
func TestAFailedQuitAfterTheMessageWasAcceptedIsNotAFailure(t *testing.T) {
// The relay said 250 to the message: retrying because QUIT misbehaved would deliver it twice.
for name, reply := range map[string]string{"hang up without a reply": "", "421 instead of 221": "421 4.4.2 closing early\r\n"} {
q, addr := startQuitDropper(t, reply)
err := (&SMTPRelay{Addr: addr}).Send(context.Background(), "b@x.org", []string{"a@x.com"}, []byte("Subject: x\r\n\r\nx\r\n"))
if err != nil {
t.Errorf("%s: Send must succeed, the message was accepted: %v", name, err)
}
q.mu.Lock()
if q.accepted != 1 {
t.Errorf("%s: accepted %d times", name, q.accepted)
}
q.mu.Unlock()
}
}
@ -279,3 +402,106 @@ func TestSingleBatchErrorsAreNotDecoratedWithBatchNumbers(t *testing.T) {
t.Fatalf("with several batches the failing range is useful: %v", err)
}
}
func TestNonASCIIAddressesAskForSMTPUTF8(t *testing.T) {
var gotUTF8 []bool
var mu sync.Mutex
backend := smtp.BackendFunc(func(*smtp.Conn) (smtp.Session, error) {
return &utf8Session{record: func(v bool) { mu.Lock(); gotUTF8 = append(gotUTF8, v); mu.Unlock() }}, nil
})
srv := smtp.NewServer(backend)
srv.EnableSMTPUTF8 = true
ln, _ := net.Listen("tcp", "127.0.0.1:0")
go srv.Serve(ln)
defer srv.Close()
r := &SMTPRelay{Addr: ln.Addr().String()}
if err := r.Send(context.Background(), "bounce@example.org", []string{"jörg@example.com"}, []byte("Subject: x\r\n\r\nx\r\n")); err != nil {
t.Fatal(err)
}
if err := r.Send(context.Background(), "bounce@example.org", []string{"plain@example.com"}, []byte("Subject: x\r\n\r\nx\r\n")); err != nil {
t.Fatal(err)
}
mu.Lock()
defer mu.Unlock()
if len(gotUTF8) != 2 || !gotUTF8[0] || gotUTF8[1] {
t.Fatalf("SMTPUTF8 only for the message that needs it: %v", gotUTF8)
}
}
type utf8Session struct{ record func(bool) }
func (s *utf8Session) Reset() {}
func (s *utf8Session) Logout() error { return nil }
func (s *utf8Session) Mail(_ string, o *smtp.MailOptions) error {
s.record(o != nil && o.UTF8)
return nil
}
func (s *utf8Session) Rcpt(string, *smtp.RcptOptions) error { return nil }
func (s *utf8Session) Data(r io.Reader) error { io.Copy(io.Discard, r); return nil }
// refuseData is a bare SMTP server that accepts MAIL and RCPT but answers the DATA command itself with reply.
func refuseData(t *testing.T, reply string) string {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { ln.Close() })
go func() {
for {
c, err := ln.Accept()
if err != nil {
return
}
go func() {
defer c.Close()
c.Write([]byte("220 test ESMTP\r\n"))
one := make([]byte, 1)
var buf []byte
for {
if _, err := c.Read(one); err != nil {
return
}
buf = append(buf, one[0])
if len(buf) < 2 || buf[len(buf)-2] != '\r' || buf[len(buf)-1] != '\n' {
continue
}
l := string(buf[:len(buf)-2])
buf = buf[:0]
switch {
case strings.HasPrefix(l, "EHLO"), strings.HasPrefix(l, "HELO"):
c.Write([]byte("250 test\r\n"))
case strings.HasPrefix(l, "MAIL"), strings.HasPrefix(l, "RCPT"):
c.Write([]byte("250 2.1.0 ok\r\n"))
case l == "DATA":
c.Write([]byte(reply))
case l == "QUIT":
c.Write([]byte("221 bye\r\n"))
return
}
}
}()
}
}()
return ln.Addr().String()
}
func TestARefusalOfTheDATACommandItselfIsNotAMissingMailbox(t *testing.T) {
// 5.1.1 here would be absurd, but a hostile or broken relay must not be able to strike members with it.
for name, tc := range map[string]struct {
reply string
perm bool
}{
"554 no valid recipients": {"554 5.5.1 no valid recipients\r\n", true},
"5.1.1 at DATA": {"550 5.1.1 whatever\r\n", true},
"451 try later": {"451 4.3.0 try later\r\n", false},
} {
err := (&SMTPRelay{Addr: refuseData(t, tc.reply)}).Send(context.Background(), "b@x.org", []string{"a@x.com"}, []byte("Subject: x\r\n\r\nx\r\n"))
if err == nil || !strings.HasPrefix(err.Error(), "DATA: ") {
t.Fatalf("%s: the error says DATA was refused: %v", name, err)
}
if IsBadMailbox(err) || IsPermanent(err) != tc.perm {
t.Errorf("%s: permanent=%v mailbox=%v", name, IsPermanent(err), IsBadMailbox(err))
}
}
}

View file

@ -251,7 +251,7 @@ func TestBadMailboxCountsAHardBounceAndDisablesAfterThree(t *testing.T) {
ms, _ := e.s.Members(l.ID)
id := ms[0].ID
e.snd.fn = func(context.Context, string, []string) error {
return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"}
return outbound.AtRcpt(&smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"})
}
for i := 1; i <= 3; i++ {
e.s.Enqueue([]byte("Subject: x\r\n\r\nx"), []store.NewOutbound{{From: "b@x.org", Rcpt: "gone@x.com", MemberID: id}})
@ -293,7 +293,7 @@ func TestOtherPermanentRejectionsNeverCountAgainstTheMember(t *testing.T) {
func TestBadMailboxForSystemMailOrUnknownMemberIsHarmless(t *testing.T) {
e := setup(t)
e.snd.fn = func(context.Context, string, []string) error {
return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"}
return outbound.AtRcpt(&smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "no such user"})
}
e.s.Enqueue([]byte("x"), []store.NewOutbound{
{From: "b@x.org", Rcpt: "a@x.com", MemberID: 0}, // confirmation mail
@ -532,3 +532,29 @@ func TestEachAttemptIsSignedAgain(t *testing.T) {
t.Fatalf("a retry is signed afresh (new timestamp), signing calls = %d", calls)
}
}
func TestARefusedSenderIsNotAMissingMailbox(t *testing.T) {
// A relay that rejects the list's own sender address (5.1.8, 5.1.7, 5.1.0 or even 5.1.1 answering
// MAIL FROM) says nothing about the recipients: the members must not be struck, and the mail must
// stay visible as failed instead of vanishing.
e := setup(t)
l, _ := e.s.CreateList("dev", "example.org", "Dev")
e.s.AddMember(l.ID, "ok@x.com", "")
ms, _ := e.s.Members(l.ID)
e.snd.fn = func(context.Context, string, []string) error {
return &outbound.PhaseError{Phase: "MAIL", Err: &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 8}, Message: "Bad outbound sender"}}
}
for i := 0; i < 5; i++ {
e.s.Enqueue([]byte("x"), []store.NewOutbound{{From: "dev+bounce-1-x@example.org", Rcpt: "ok@x.com", MemberID: ms[0].ID}})
st, _ := e.w.RunOnce(context.Background())
if st.Failed != 1 || st.Bounced != 0 || st.Dropped != 0 {
t.Fatalf("round %d: %+v", i+1, st)
}
}
if m, _ := e.s.MemberByID(ms[0].ID); m.BounceCount != 0 || m.Disabled {
t.Fatalf("the member was struck for our own sender problem: %+v", m)
}
if stats, _ := e.s.OutboxStats(); stats.Failed != 5 {
t.Fatalf("every delivery stays listed as failed: %+v", stats)
}
}

View file

@ -138,3 +138,16 @@ func TestSimultaneousApprovalsDeliverOnce(t *testing.T) {
t.Fatalf("2 members, one approval: pending = %d", st.Pending)
}
}
func TestAMessageTooBigToHoldIsRefused(t *testing.T) {
e := newEnv(t)
e.s.SetPolicies(e.list.ID, "moderated", "open", "private")
big := append(postFrom("alice@example.com"), make([]byte, 10<<20)...)
_, err := e.r.Route("alice@example.com", "dev@example.org", big)
if !errors.Is(err, ErrHeldTooLarge) {
t.Fatalf("want ErrHeldTooLarge, got %v", err)
}
if held, _ := e.s.ListHeld(e.list.ID); len(held) != 0 {
t.Fatal("nothing is held")
}
}

View file

@ -22,6 +22,8 @@ var (
ErrDiscarded = errors.New("discarded")
// ErrHeldFull means the list's moderation queue is full: refuse for now, the sender may retry.
ErrHeldFull = store.ErrHeldFull
// ErrHeldTooLarge means the message is too big to hold for moderation on a moderated list.
ErrHeldTooLarge = store.ErrHeldTooLarge
// ErrNotMember means the sender may not post to the list.
ErrNotMember = errors.New("sender is not a member")
// ErrHeld means the message was queued for moderation. It is not a failure.

View file

@ -445,3 +445,27 @@ func TestThreadRootsFromTheNewestEndOfAChain(t *testing.T) {
t.Fatalf("roots: %v", forward)
}
}
func TestThreadListingsAreOldestFirst(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
late := msg("<late@x>", "Late reply", "b")
late.Date = time.Unix(1_700_000_900, 0)
late.InReplyTo = "<root@x>"
early := msg("<early@x>", "Early reply", "b")
early.Date = time.Unix(1_700_000_100, 0)
early.InReplyTo = "<root@x>"
root := msg("<root@x>", "Root", "b")
root.Date = time.Unix(1_700_000_000, 0)
rootID, _, _ := s.ArchiveMessage(l.ID, root)
s.ArchiveMessage(l.ID, late) // inserted before the earlier reply
s.ArchiveMessage(l.ID, early)
th, _ := s.ThreadMessages(l.ID, rootID)
got := []string{}
for _, m := range th {
got = append(got, m.Subject)
}
if strings.Join(got, "|") != "Root|Early reply|Late reply" {
t.Fatalf("a thread reads in the order it happened, not in the order it was stored: %v", got)
}
}

View file

@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"path/filepath"
"strings"
"testing"
"time"
)
@ -326,3 +327,143 @@ func TestOldHeldMessagesExpire(t *testing.T) {
t.Fatalf("expiry is applied to the list being written to, not to others: %v", err)
}
}
func TestHeldListingsCarryASummaryAndNeverTheBody(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
body := strings.Repeat("0123456789abcdef\r\n", 1000)
data := []byte("From: =?UTF-8?Q?Jos=C3=A9?= <jose@example.com>\r\nSubject: =?UTF-8?Q?Caff=C3=A8_a_las_nueve?=\r\n\r\n" + body)
id, err := s.HoldMessage(l.ID, "Alice@Example.com", data, "r")
if err != nil {
t.Fatal(err)
}
list, err := s.ListHeld(l.ID)
if err != nil || len(list) != 1 {
t.Fatalf("%v %v", list, err)
}
h := list[0]
if h.Data != nil {
t.Fatalf("a listing must not read the %d-byte body into memory", len(data))
}
if h.Subject != "Caffè a las nueve" || h.FromHeader != "=?UTF-8?Q?Jos=C3=A9?= <jose@example.com>" || h.Size != len(data) || h.Sender != "alice@example.com" {
t.Fatalf("summary: %+v", h)
}
full, err := s.GetHeld(id)
if err != nil || len(full.Data) != len(data) || full.Subject != h.Subject {
t.Fatalf("a single message still comes with its body: %v %d", err, len(full.Data))
}
}
func TestHeldSummaryOfOddMessages(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
long := strings.Repeat("x", 500)
for name, tc := range map[string]struct {
data string
subject, from string
}{
"no headers": {"\r\njust a body\r\n", "", ""},
"empty": {"", "", ""},
"no subject": {"From: a@x.com\r\n\r\nb", "", "a@x.com"},
"long subject": {"Subject: " + long + "\r\n\r\nb", strings.Repeat("x", 200), ""},
"long from": {"From: " + long + "\r\n\r\nb", "", strings.Repeat("x", 200)},
"folded subject": {"Subject: one\r\n two\r\n\r\nb", "one two", ""},
"control characters": {"Subject: a\x01b\x1b[31mred\r\n\r\nb", "ab[31mred", ""},
} {
id, err := s.HoldMessage(l.ID, "a@x.com", []byte(tc.data), "")
if err != nil {
t.Fatalf("%s: %v", name, err)
}
h, _ := s.GetHeld(id)
if h.Subject != tc.subject || h.FromHeader != tc.from {
t.Errorf("%s: subject %q from %q, want %q %q", name, h.Subject, h.FromHeader, tc.subject, tc.from)
}
}
}
func TestHeldQueueHasAByteCapAndAMessageCap(t *testing.T) {
oldMsg, oldList := heldMessageLimit, heldBytesLimit
heldMessageLimit, heldBytesLimit = 1000, 3500
t.Cleanup(func() { heldMessageLimit, heldBytesLimit = oldMsg, oldList })
s := newTestStore(t)
a, _ := s.CreateList("a", "example.org", "A")
b, _ := s.CreateList("b", "example.org", "B")
msg := func(n int) []byte { return []byte("Subject: s\r\n\r\n" + strings.Repeat("x", n-14)) }
if _, err := s.HoldMessage(a.ID, "x@y.com", msg(1001), ""); !errors.Is(err, ErrHeldTooLarge) {
t.Fatalf("one message over the limit: want ErrHeldTooLarge, got %v", err)
}
if _, err := s.HoldMessage(a.ID, "x@y.com", msg(1000), ""); err != nil {
t.Fatalf("exactly at the limit is fine: %v", err)
}
var ids []int64
for i := 0; i < 2; i++ { // 1000 + 1000 + 1000 = 3000 of 3500
id, err := s.HoldMessage(a.ID, "x@y.com", msg(1000), "")
if err != nil {
t.Fatal(err)
}
ids = append(ids, id)
}
if _, err := s.HoldMessage(a.ID, "x@y.com", msg(600), ""); !errors.Is(err, ErrHeldFull) {
t.Fatalf("3000 + 600 > 3500: want ErrHeldFull, got %v", err)
}
if _, err := s.HoldMessage(a.ID, "x@y.com", msg(500), ""); err != nil {
t.Fatalf("3000 + 500 = 3500 fits: %v", err)
}
if _, err := s.HoldMessage(b.ID, "x@y.com", msg(1000), ""); err != nil {
t.Fatalf("another list has its own allowance: %v", err)
}
if err := s.DeleteHeld(ids[0]); err != nil {
t.Fatal(err)
}
if _, err := s.HoldMessage(a.ID, "x@y.com", msg(900), ""); err != nil {
t.Fatalf("moderating frees room: %v", err)
}
}
func TestMigrationSummarisesExistingHeldMessages(t *testing.T) {
path := t.TempDir() + "/old.db"
s, _ := Open(path)
l, _ := s.CreateList("dev", "example.org", "Dev")
data := []byte("From: Bob <bob@x.com>\r\nSubject: Old held\r\n\r\nbody")
id, _ := s.HoldMessage(l.ID, "bob@x.com", data, "r")
s.db.Exec(`ALTER TABLE held DROP COLUMN subject`)
s.db.Exec(`ALTER TABLE held DROP COLUMN from_header`)
s.db.Exec(`ALTER TABLE held DROP COLUMN size`)
s.Close()
s2, err := Open(path)
if err != nil {
t.Fatalf("migrate: %v", err)
}
defer s2.Close()
list, _ := s2.ListHeld(l.ID)
if len(list) != 1 || list[0].ID != id || list[0].Subject != "Old held" || list[0].FromHeader != "Bob <bob@x.com>" || list[0].Size != len(data) || list[0].Data != nil {
t.Fatalf("an existing held message is summarised by the migration: %+v", list)
}
}
func TestThreadListingsDoNotLoadBodies(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
root, _, _ := s.ArchiveMessage(l.ID, msg("<r@x>", "Root", strings.Repeat("big body ", 1000)))
reply := msg("<c@x>", "Re: Root", "reply body")
reply.InReplyTo = "<r@x>"
s.ArchiveMessage(l.ID, reply)
th, err := s.ThreadMessages(l.ID, root)
if err != nil || len(th) != 2 {
t.Fatalf("%v %v", th, err)
}
for _, m := range th {
if m.Body != "" {
t.Fatalf("a thread listing needs subjects, not bodies: %q", m.Body[:20])
}
if m.Size == 0 || m.Subject == "" {
t.Fatalf("summary: %+v", m)
}
}
one, _ := s.GetMessage(root)
if !strings.HasPrefix(one.Body, "big body") {
t.Fatal("a single message still has its body")
}
}

View file

@ -2,13 +2,17 @@
package store
import (
"bytes"
"crypto/rand"
"database/sql"
"encoding/hex"
"errors"
"fmt"
"mime"
"net/mail"
"strings"
"time"
"unicode"
_ "modernc.org/sqlite"
)
@ -18,6 +22,8 @@ var (
ErrNotFound = errors.New("not found")
// ErrHeldFull is returned when a list's moderation queue has no more room.
ErrHeldFull = errors.New("moderation queue is full")
// ErrHeldTooLarge is returned for a message too big to be held for moderation.
ErrHeldTooLarge = errors.New("message too large to hold for moderation")
// ErrExists is returned when creating something that already exists.
ErrExists = errors.New("already exists")
)
@ -43,6 +49,12 @@ const (
ActionLeave = "leave"
)
// Limits on the moderation queue of one list. They are variables so the tests can lower them.
var (
heldMessageLimit = 10 << 20 // bytes in one held message
heldBytesLimit = 200 << 20 // bytes held for one list, all messages together
)
// BounceThreshold is the number of hard bounces after which a member is disabled.
const BounceThreshold = 3
@ -94,6 +106,11 @@ type Held struct {
Data []byte
CreatedAt time.Time
Reason string // why the message was held, for the admin
// What a listing needs, stored when the message is held so that a listing never has to read a body.
Subject string // decoded, at most 200 characters
FromHeader string // the From header as received, at most 200 characters
Size int // bytes of the message
}
// Pending is an unconfirmed subscribe or unsubscribe request.
@ -137,7 +154,10 @@ CREATE TABLE IF NOT EXISTS held (
sender TEXT NOT NULL,
data BLOB NOT NULL,
created_at INTEGER NOT NULL,
reason TEXT NOT NULL DEFAULT ''
reason TEXT NOT NULL DEFAULT '',
subject TEXT NOT NULL DEFAULT '',
from_header TEXT NOT NULL DEFAULT '',
size INTEGER NOT NULL DEFAULT 0
);
CREATE TABLE IF NOT EXISTS pending (
token TEXT PRIMARY KEY,
@ -275,11 +295,82 @@ func migrate(db *sql.DB) error {
return err
}
}
if has, _ := hasColumn(db, "held", "size"); !has {
if err := addHeldSummary(db); err != nil {
return err
}
}
}
_, err := db.Exec(schema)
return err
}
// addHeldSummary adds the summary columns of the held queue and fills them for the messages
// already there, one message at a time so that a big queue is not loaded into memory.
func addHeldSummary(db *sql.DB) error {
for _, stmt := range []string{
`ALTER TABLE held ADD COLUMN subject TEXT NOT NULL DEFAULT ''`,
`ALTER TABLE held ADD COLUMN from_header TEXT NOT NULL DEFAULT ''`,
`ALTER TABLE held ADD COLUMN size INTEGER NOT NULL DEFAULT 0`,
`UPDATE held SET size=length(data)`,
} {
if _, err := db.Exec(stmt); err != nil {
return err
}
}
rows, err := db.Query(`SELECT id FROM held`)
if err != nil {
return err
}
var ids []int64
for rows.Next() {
var id int64
if err := rows.Scan(&id); err != nil {
rows.Close()
return err
}
ids = append(ids, id)
}
rows.Close()
for _, id := range ids {
var data []byte
if err := db.QueryRow(`SELECT data FROM held WHERE id=?`, id).Scan(&data); err != nil {
return err
}
subject, from := summarize(data)
if _, err := db.Exec(`UPDATE held SET subject=?, from_header=? WHERE id=?`, subject, from, id); err != nil {
return err
}
}
return nil
}
// summarize extracts what a moderation listing shows: the decoded subject and the From header, both
// without control characters and cut to 200 characters.
func summarize(data []byte) (subject, from string) {
m, err := mail.ReadMessage(bytes.NewReader(data))
if err != nil {
return "", ""
}
clean := func(s string) string {
s = strings.Map(func(r rune) rune {
if unicode.IsControl(r) {
return -1
}
return r
}, strings.Join(strings.Fields(s), " "))
if r := []rune(s); len(r) > 200 {
s = string(r[:200])
}
return s
}
raw := m.Header.Get("Subject")
if dec, err := new(mime.WordDecoder).DecodeHeader(raw); err == nil {
raw = dec
}
return clean(raw), clean(m.Header.Get("From"))
}
// Close closes the database.
func (s *Store) Close() error { return s.db.Close() }
@ -468,34 +559,47 @@ func (s *Store) RecordHardBounce(memberID int64, threshold int) (bool, error) {
// Held messages.
// HoldMessage queues a message for moderation and returns its id. reason tells the admin why.
// HoldMessage queues a message for moderation and returns its id. reason tells the admin why. A list
// holds a bounded amount: ErrHeldTooLarge for one message over heldMessageLimit, ErrHeldFull when the
// number of messages or the bytes held for the list would be exceeded.
func (s *Store) HoldMessage(listID int64, sender string, data []byte, reason string) (int64, error) {
if data == nil {
data = []byte{} // the column is NOT NULL
}
if len(data) > heldMessageLimit {
return 0, ErrHeldTooLarge
}
now := s.Now()
if _, err := s.db.Exec(`DELETE FROM held WHERE list_id=? AND created_at<?`, listID, now.Add(-heldTTL).Unix()); err != nil {
return 0, err
}
var n int
if err := s.db.QueryRow(`SELECT COUNT(*) FROM held WHERE list_id=?`, listID).Scan(&n); err != nil {
var n, bytesHeld int
if err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(SUM(size),0) FROM held WHERE list_id=?`, listID).Scan(&n, &bytesHeld); err != nil {
return 0, err
}
if n >= maxHeldPerList {
if n >= maxHeldPerList || bytesHeld+len(data) > heldBytesLimit {
return 0, ErrHeldFull
}
res, err := s.db.Exec(`INSERT INTO held(list_id, sender, data, created_at, reason) VALUES(?,?,?,?,?)`,
listID, norm(sender), data, now.Unix(), reason)
subject, from := summarize(data)
res, err := s.db.Exec(`INSERT INTO held(list_id, sender, data, created_at, reason, subject, from_header, size) VALUES(?,?,?,?,?,?,?,?)`,
listID, norm(sender), data, now.Unix(), reason, subject, from, len(data))
if err != nil {
return 0, err
}
return res.LastInsertId()
}
func scanHeld(row interface{ Scan(...any) error }) (Held, error) {
const heldCols = `id, list_id, sender, created_at, reason, subject, from_header, size`
func scanHeld(row interface{ Scan(...any) error }, withData bool) (Held, error) {
var h Held
var ts int64
err := row.Scan(&h.ID, &h.ListID, &h.Sender, &h.Data, &ts, &h.Reason)
var err error
if withData {
err = row.Scan(&h.ID, &h.ListID, &h.Sender, &ts, &h.Reason, &h.Subject, &h.FromHeader, &h.Size, &h.Data)
} else {
err = row.Scan(&h.ID, &h.ListID, &h.Sender, &ts, &h.Reason, &h.Subject, &h.FromHeader, &h.Size)
}
if errors.Is(err, sql.ErrNoRows) {
return Held{}, ErrNotFound
}
@ -503,16 +607,17 @@ func scanHeld(row interface{ Scan(...any) error }) (Held, error) {
return h, err
}
// ListHeld returns the held messages of a list, oldest first.
// ListHeld returns the held messages of a list, oldest first. It carries the summary of each message
// and never its body (Data is nil): a listing must not read up to 25 MiB per message into memory.
func (s *Store) ListHeld(listID int64) ([]Held, error) {
rows, err := s.db.Query(`SELECT id, list_id, sender, data, created_at, reason FROM held WHERE list_id=? ORDER BY id`, listID)
rows, err := s.db.Query(`SELECT `+heldCols+` FROM held WHERE list_id=? ORDER BY id`, listID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Held
for rows.Next() {
h, err := scanHeld(rows)
h, err := scanHeld(rows, false)
if err != nil {
return nil, err
}
@ -521,9 +626,9 @@ func (s *Store) ListHeld(listID int64) ([]Held, error) {
return out, rows.Err()
}
// GetHeld returns a held message by id.
// GetHeld returns one held message with its body.
func (s *Store) GetHeld(id int64) (Held, error) {
return scanHeld(s.db.QueryRow(`SELECT id, list_id, sender, data, created_at, reason FROM held WHERE id=?`, id))
return scanHeld(s.db.QueryRow(`SELECT `+heldCols+`, data FROM held WHERE id=?`, id), true)
}
// DeleteHeld discards a held message.
@ -885,20 +990,8 @@ func (s *Store) SearchMessages(listID int64, query string, limit, offset int) ([
return out, total, err
}
// ThreadMessages returns every message of a thread, oldest first, with bodies.
// ThreadMessages returns every message of a thread, oldest first, without bodies: a thread listing
// shows subjects, and a thread can hold many messages of up to 1 MiB of text each.
func (s *Store) ThreadMessages(listID, threadID int64) ([]Message, error) {
rows, err := s.db.Query(`SELECT `+msgCols+`, body FROM archive WHERE list_id=? AND thread_id=? ORDER BY date, id`, listID, threadID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Message
for rows.Next() {
m, err := scanMessage(rows, true)
if err != nil {
return nil, err
}
out = append(out, m)
}
return out, rows.Err()
return s.queryMessages(`SELECT `+msgCols+` FROM archive WHERE list_id=? AND thread_id=? ORDER BY date, id`, listID, threadID)
}