fix(store): moderation listings never read message bodies, bound the held bytes

- Store subject, From header and size when a message is held; ListHeld returns the summary only
- Cap one held message at 10 MiB (ErrHeldTooLarge) and the bytes held per list at 200 MiB (ErrHeldFull)
- Summarise the messages already held when migrating an existing database, one at a time
- Show the From header and whether it differs from the envelope sender in the held API
- Thread listings no longer load the bodies of the thread
This commit is contained in:
tgiachi 2026-11-10 11:39:21 +01:00
commit 94b5a58e93
4 changed files with 310 additions and 47 deletions

View file

@ -2,13 +2,11 @@
package api
import (
"bytes"
"context"
"crypto/subtle"
"encoding/json"
"errors"
"fmt"
"mime"
"net/http"
"net/mail"
"strconv"
@ -353,18 +351,17 @@ type heldDTO struct {
Size int `json:"size"`
CreatedAt time.Time `json:"created_at"`
Reason string `json:"reason"` // why the message was held
// The From header as received, and whether it names someone other than the envelope sender:
// what recipients will see as the author, against who really sent it.
From string `json:"from"`
Mismatch bool `json:"mismatch"`
}
func subjectOf(data []byte) string {
m, err := mail.ReadMessage(bytes.NewReader(data))
if err != nil {
return ""
}
subj := m.Header.Get("Subject")
if dec, err := new(mime.WordDecoder).DecodeHeader(subj); err == nil {
return dec
}
return subj
// fromMismatch reports whether the From header names an address other than the envelope sender, or
// no usable address at all.
func fromMismatch(fromHeader, sender string) bool {
a, err := mail.ParseAddress(fromHeader)
return err != nil || !strings.EqualFold(a.Address, sender)
}
func (s *server) listHeld(w http.ResponseWriter, r *http.Request) {
@ -383,7 +380,8 @@ func (s *server) listHeld(w http.ResponseWriter, r *http.Request) {
}
out := make([]heldDTO, 0, len(hs))
for _, h := range hs {
out = append(out, heldDTO{ID: h.ID, Sender: h.Sender, Subject: subjectOf(h.Data), Size: len(h.Data), CreatedAt: h.CreatedAt, Reason: h.Reason})
out = append(out, heldDTO{ID: h.ID, Sender: h.Sender, Subject: h.Subject, Size: h.Size, CreatedAt: h.CreatedAt, Reason: h.Reason,
From: h.FromHeader, Mismatch: fromMismatch(h.FromHeader, h.Sender)})
}
writeJSON(w, http.StatusOK, out)
}

View file

@ -271,11 +271,13 @@ func heldServer(t *testing.T) (http.Handler, *store.Store, *fakeReleaser) {
}
type heldJSON struct {
ID int64 `json:"id"`
Sender string `json:"sender"`
Subject string `json:"subject"`
Size int `json:"size"`
Reason string `json:"reason"`
ID int64 `json:"id"`
Sender string `json:"sender"`
From string `json:"from"`
Mismatch bool `json:"mismatch"`
Subject string `json:"subject"`
Size int `json:"size"`
Reason string `json:"reason"`
}
func TestHeldQueueAPI(t *testing.T) {
@ -298,6 +300,35 @@ func TestHeldQueueAPI(t *testing.T) {
expect(t, do(t, h, "DELETE", "/api/v1/held/abc", "", true), 404)
}
func TestHeldListingShowsWhoReallyWrote(t *testing.T) {
s, _ := store.Open(":memory:")
defer s.Close()
l, _ := s.CreateList("dev", "example.org", "Dev")
s.HoldMessage(l.ID, "bob@x.com", []byte("From: Bob <BOB@x.com>\r\nSubject: honest\r\n\r\nb"), "")
s.HoldMessage(l.ID, "bob@x.com", []byte("From: Eve <eve@x.com>\r\nSubject: passing as Eve\r\n\r\nb"), "")
s.HoldMessage(l.ID, "bob@x.com", []byte("Subject: no From at all\r\n\r\nb"), "")
s.HoldMessage(l.ID, "bob@x.com", []byte("From: not an address\r\nSubject: garbled\r\n\r\nb"), "")
h := New(s, token, nil)
var hs []heldJSON
decode(t, do(t, h, "GET", "/api/v1/lists/1/held", "", true), &hs)
if len(hs) != 4 {
t.Fatalf("%+v", hs)
}
want := []struct {
from string
mismatch bool
}{{"Bob <BOB@x.com>", false}, {"Eve <eve@x.com>", true}, {"", true}, {"not an address", true}}
for i, w := range want {
if hs[i].From != w.from || hs[i].Mismatch != w.mismatch {
t.Errorf("held %d: from %q mismatch %v, want %q %v", i+1, hs[i].From, hs[i].Mismatch, w.from, w.mismatch)
}
if hs[i].Sender != "bob@x.com" {
t.Errorf("the envelope sender stays visible: %q", hs[i].Sender)
}
}
}
func TestHeldRequiresAuth(t *testing.T) {
h, _, _ := heldServer(t)
expect(t, do(t, h, "POST", "/api/v1/held/1/approve", "", false), 401)

View file

@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"path/filepath"
"strings"
"testing"
"time"
)
@ -326,3 +327,143 @@ func TestOldHeldMessagesExpire(t *testing.T) {
t.Fatalf("expiry is applied to the list being written to, not to others: %v", err)
}
}
func TestHeldListingsCarryASummaryAndNeverTheBody(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
body := strings.Repeat("0123456789abcdef\r\n", 1000)
data := []byte("From: =?UTF-8?Q?Jos=C3=A9?= <jose@example.com>\r\nSubject: =?UTF-8?Q?Caff=C3=A8_a_las_nueve?=\r\n\r\n" + body)
id, err := s.HoldMessage(l.ID, "Alice@Example.com", data, "r")
if err != nil {
t.Fatal(err)
}
list, err := s.ListHeld(l.ID)
if err != nil || len(list) != 1 {
t.Fatalf("%v %v", list, err)
}
h := list[0]
if h.Data != nil {
t.Fatalf("a listing must not read the %d-byte body into memory", len(data))
}
if h.Subject != "Caffè a las nueve" || h.FromHeader != "=?UTF-8?Q?Jos=C3=A9?= <jose@example.com>" || h.Size != len(data) || h.Sender != "alice@example.com" {
t.Fatalf("summary: %+v", h)
}
full, err := s.GetHeld(id)
if err != nil || len(full.Data) != len(data) || full.Subject != h.Subject {
t.Fatalf("a single message still comes with its body: %v %d", err, len(full.Data))
}
}
func TestHeldSummaryOfOddMessages(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
long := strings.Repeat("x", 500)
for name, tc := range map[string]struct {
data string
subject, from string
}{
"no headers": {"\r\njust a body\r\n", "", ""},
"empty": {"", "", ""},
"no subject": {"From: a@x.com\r\n\r\nb", "", "a@x.com"},
"long subject": {"Subject: " + long + "\r\n\r\nb", strings.Repeat("x", 200), ""},
"long from": {"From: " + long + "\r\n\r\nb", "", strings.Repeat("x", 200)},
"folded subject": {"Subject: one\r\n two\r\n\r\nb", "one two", ""},
"control characters": {"Subject: a\x01b\x1b[31mred\r\n\r\nb", "ab[31mred", ""},
} {
id, err := s.HoldMessage(l.ID, "a@x.com", []byte(tc.data), "")
if err != nil {
t.Fatalf("%s: %v", name, err)
}
h, _ := s.GetHeld(id)
if h.Subject != tc.subject || h.FromHeader != tc.from {
t.Errorf("%s: subject %q from %q, want %q %q", name, h.Subject, h.FromHeader, tc.subject, tc.from)
}
}
}
func TestHeldQueueHasAByteCapAndAMessageCap(t *testing.T) {
oldMsg, oldList := heldMessageLimit, heldBytesLimit
heldMessageLimit, heldBytesLimit = 1000, 3500
t.Cleanup(func() { heldMessageLimit, heldBytesLimit = oldMsg, oldList })
s := newTestStore(t)
a, _ := s.CreateList("a", "example.org", "A")
b, _ := s.CreateList("b", "example.org", "B")
msg := func(n int) []byte { return []byte("Subject: s\r\n\r\n" + strings.Repeat("x", n-14)) }
if _, err := s.HoldMessage(a.ID, "x@y.com", msg(1001), ""); !errors.Is(err, ErrHeldTooLarge) {
t.Fatalf("one message over the limit: want ErrHeldTooLarge, got %v", err)
}
if _, err := s.HoldMessage(a.ID, "x@y.com", msg(1000), ""); err != nil {
t.Fatalf("exactly at the limit is fine: %v", err)
}
var ids []int64
for i := 0; i < 2; i++ { // 1000 + 1000 + 1000 = 3000 of 3500
id, err := s.HoldMessage(a.ID, "x@y.com", msg(1000), "")
if err != nil {
t.Fatal(err)
}
ids = append(ids, id)
}
if _, err := s.HoldMessage(a.ID, "x@y.com", msg(600), ""); !errors.Is(err, ErrHeldFull) {
t.Fatalf("3000 + 600 > 3500: want ErrHeldFull, got %v", err)
}
if _, err := s.HoldMessage(a.ID, "x@y.com", msg(500), ""); err != nil {
t.Fatalf("3000 + 500 = 3500 fits: %v", err)
}
if _, err := s.HoldMessage(b.ID, "x@y.com", msg(1000), ""); err != nil {
t.Fatalf("another list has its own allowance: %v", err)
}
if err := s.DeleteHeld(ids[0]); err != nil {
t.Fatal(err)
}
if _, err := s.HoldMessage(a.ID, "x@y.com", msg(900), ""); err != nil {
t.Fatalf("moderating frees room: %v", err)
}
}
func TestMigrationSummarisesExistingHeldMessages(t *testing.T) {
path := t.TempDir() + "/old.db"
s, _ := Open(path)
l, _ := s.CreateList("dev", "example.org", "Dev")
data := []byte("From: Bob <bob@x.com>\r\nSubject: Old held\r\n\r\nbody")
id, _ := s.HoldMessage(l.ID, "bob@x.com", data, "r")
s.db.Exec(`ALTER TABLE held DROP COLUMN subject`)
s.db.Exec(`ALTER TABLE held DROP COLUMN from_header`)
s.db.Exec(`ALTER TABLE held DROP COLUMN size`)
s.Close()
s2, err := Open(path)
if err != nil {
t.Fatalf("migrate: %v", err)
}
defer s2.Close()
list, _ := s2.ListHeld(l.ID)
if len(list) != 1 || list[0].ID != id || list[0].Subject != "Old held" || list[0].FromHeader != "Bob <bob@x.com>" || list[0].Size != len(data) || list[0].Data != nil {
t.Fatalf("an existing held message is summarised by the migration: %+v", list)
}
}
func TestThreadListingsDoNotLoadBodies(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
root, _, _ := s.ArchiveMessage(l.ID, msg("<r@x>", "Root", strings.Repeat("big body ", 1000)))
reply := msg("<c@x>", "Re: Root", "reply body")
reply.InReplyTo = "<r@x>"
s.ArchiveMessage(l.ID, reply)
th, err := s.ThreadMessages(l.ID, root)
if err != nil || len(th) != 2 {
t.Fatalf("%v %v", th, err)
}
for _, m := range th {
if m.Body != "" {
t.Fatalf("a thread listing needs subjects, not bodies: %q", m.Body[:20])
}
if m.Size == 0 || m.Subject == "" {
t.Fatalf("summary: %+v", m)
}
}
one, _ := s.GetMessage(root)
if !strings.HasPrefix(one.Body, "big body") {
t.Fatal("a single message still has its body")
}
}

View file

@ -2,13 +2,17 @@
package store
import (
"bytes"
"crypto/rand"
"database/sql"
"encoding/hex"
"errors"
"fmt"
"mime"
"net/mail"
"strings"
"time"
"unicode"
_ "modernc.org/sqlite"
)
@ -18,6 +22,8 @@ var (
ErrNotFound = errors.New("not found")
// ErrHeldFull is returned when a list's moderation queue has no more room.
ErrHeldFull = errors.New("moderation queue is full")
// ErrHeldTooLarge is returned for a message too big to be held for moderation.
ErrHeldTooLarge = errors.New("message too large to hold for moderation")
// ErrExists is returned when creating something that already exists.
ErrExists = errors.New("already exists")
)
@ -43,6 +49,12 @@ const (
ActionLeave = "leave"
)
// Limits on the moderation queue of one list. They are variables so the tests can lower them.
var (
heldMessageLimit = 10 << 20 // bytes in one held message
heldBytesLimit = 200 << 20 // bytes held for one list, all messages together
)
// BounceThreshold is the number of hard bounces after which a member is disabled.
const BounceThreshold = 3
@ -94,6 +106,11 @@ type Held struct {
Data []byte
CreatedAt time.Time
Reason string // why the message was held, for the admin
// What a listing needs, stored when the message is held so that a listing never has to read a body.
Subject string // decoded, at most 200 characters
FromHeader string // the From header as received, at most 200 characters
Size int // bytes of the message
}
// Pending is an unconfirmed subscribe or unsubscribe request.
@ -137,7 +154,10 @@ CREATE TABLE IF NOT EXISTS held (
sender TEXT NOT NULL,
data BLOB NOT NULL,
created_at INTEGER NOT NULL,
reason TEXT NOT NULL DEFAULT ''
reason TEXT NOT NULL DEFAULT '',
subject TEXT NOT NULL DEFAULT '',
from_header TEXT NOT NULL DEFAULT '',
size INTEGER NOT NULL DEFAULT 0
);
CREATE TABLE IF NOT EXISTS pending (
token TEXT PRIMARY KEY,
@ -275,11 +295,82 @@ func migrate(db *sql.DB) error {
return err
}
}
if has, _ := hasColumn(db, "held", "size"); !has {
if err := addHeldSummary(db); err != nil {
return err
}
}
}
_, err := db.Exec(schema)
return err
}
// addHeldSummary adds the summary columns of the held queue and fills them for the messages
// already there, one message at a time so that a big queue is not loaded into memory.
func addHeldSummary(db *sql.DB) error {
for _, stmt := range []string{
`ALTER TABLE held ADD COLUMN subject TEXT NOT NULL DEFAULT ''`,
`ALTER TABLE held ADD COLUMN from_header TEXT NOT NULL DEFAULT ''`,
`ALTER TABLE held ADD COLUMN size INTEGER NOT NULL DEFAULT 0`,
`UPDATE held SET size=length(data)`,
} {
if _, err := db.Exec(stmt); err != nil {
return err
}
}
rows, err := db.Query(`SELECT id FROM held`)
if err != nil {
return err
}
var ids []int64
for rows.Next() {
var id int64
if err := rows.Scan(&id); err != nil {
rows.Close()
return err
}
ids = append(ids, id)
}
rows.Close()
for _, id := range ids {
var data []byte
if err := db.QueryRow(`SELECT data FROM held WHERE id=?`, id).Scan(&data); err != nil {
return err
}
subject, from := summarize(data)
if _, err := db.Exec(`UPDATE held SET subject=?, from_header=? WHERE id=?`, subject, from, id); err != nil {
return err
}
}
return nil
}
// summarize extracts what a moderation listing shows: the decoded subject and the From header, both
// without control characters and cut to 200 characters.
func summarize(data []byte) (subject, from string) {
m, err := mail.ReadMessage(bytes.NewReader(data))
if err != nil {
return "", ""
}
clean := func(s string) string {
s = strings.Map(func(r rune) rune {
if unicode.IsControl(r) {
return -1
}
return r
}, strings.Join(strings.Fields(s), " "))
if r := []rune(s); len(r) > 200 {
s = string(r[:200])
}
return s
}
raw := m.Header.Get("Subject")
if dec, err := new(mime.WordDecoder).DecodeHeader(raw); err == nil {
raw = dec
}
return clean(raw), clean(m.Header.Get("From"))
}
// Close closes the database.
func (s *Store) Close() error { return s.db.Close() }
@ -468,34 +559,47 @@ func (s *Store) RecordHardBounce(memberID int64, threshold int) (bool, error) {
// Held messages.
// HoldMessage queues a message for moderation and returns its id. reason tells the admin why.
// HoldMessage queues a message for moderation and returns its id. reason tells the admin why. A list
// holds a bounded amount: ErrHeldTooLarge for one message over heldMessageLimit, ErrHeldFull when the
// number of messages or the bytes held for the list would be exceeded.
func (s *Store) HoldMessage(listID int64, sender string, data []byte, reason string) (int64, error) {
if data == nil {
data = []byte{} // the column is NOT NULL
}
if len(data) > heldMessageLimit {
return 0, ErrHeldTooLarge
}
now := s.Now()
if _, err := s.db.Exec(`DELETE FROM held WHERE list_id=? AND created_at<?`, listID, now.Add(-heldTTL).Unix()); err != nil {
return 0, err
}
var n int
if err := s.db.QueryRow(`SELECT COUNT(*) FROM held WHERE list_id=?`, listID).Scan(&n); err != nil {
var n, bytesHeld int
if err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(SUM(size),0) FROM held WHERE list_id=?`, listID).Scan(&n, &bytesHeld); err != nil {
return 0, err
}
if n >= maxHeldPerList {
if n >= maxHeldPerList || bytesHeld+len(data) > heldBytesLimit {
return 0, ErrHeldFull
}
res, err := s.db.Exec(`INSERT INTO held(list_id, sender, data, created_at, reason) VALUES(?,?,?,?,?)`,
listID, norm(sender), data, now.Unix(), reason)
subject, from := summarize(data)
res, err := s.db.Exec(`INSERT INTO held(list_id, sender, data, created_at, reason, subject, from_header, size) VALUES(?,?,?,?,?,?,?,?)`,
listID, norm(sender), data, now.Unix(), reason, subject, from, len(data))
if err != nil {
return 0, err
}
return res.LastInsertId()
}
func scanHeld(row interface{ Scan(...any) error }) (Held, error) {
const heldCols = `id, list_id, sender, created_at, reason, subject, from_header, size`
func scanHeld(row interface{ Scan(...any) error }, withData bool) (Held, error) {
var h Held
var ts int64
err := row.Scan(&h.ID, &h.ListID, &h.Sender, &h.Data, &ts, &h.Reason)
var err error
if withData {
err = row.Scan(&h.ID, &h.ListID, &h.Sender, &ts, &h.Reason, &h.Subject, &h.FromHeader, &h.Size, &h.Data)
} else {
err = row.Scan(&h.ID, &h.ListID, &h.Sender, &ts, &h.Reason, &h.Subject, &h.FromHeader, &h.Size)
}
if errors.Is(err, sql.ErrNoRows) {
return Held{}, ErrNotFound
}
@ -503,16 +607,17 @@ func scanHeld(row interface{ Scan(...any) error }) (Held, error) {
return h, err
}
// ListHeld returns the held messages of a list, oldest first.
// ListHeld returns the held messages of a list, oldest first. It carries the summary of each message
// and never its body (Data is nil): a listing must not read up to 25 MiB per message into memory.
func (s *Store) ListHeld(listID int64) ([]Held, error) {
rows, err := s.db.Query(`SELECT id, list_id, sender, data, created_at, reason FROM held WHERE list_id=? ORDER BY id`, listID)
rows, err := s.db.Query(`SELECT `+heldCols+` FROM held WHERE list_id=? ORDER BY id`, listID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Held
for rows.Next() {
h, err := scanHeld(rows)
h, err := scanHeld(rows, false)
if err != nil {
return nil, err
}
@ -521,9 +626,9 @@ func (s *Store) ListHeld(listID int64) ([]Held, error) {
return out, rows.Err()
}
// GetHeld returns a held message by id.
// GetHeld returns one held message with its body.
func (s *Store) GetHeld(id int64) (Held, error) {
return scanHeld(s.db.QueryRow(`SELECT id, list_id, sender, data, created_at, reason FROM held WHERE id=?`, id))
return scanHeld(s.db.QueryRow(`SELECT `+heldCols+`, data FROM held WHERE id=?`, id), true)
}
// DeleteHeld discards a held message.
@ -885,20 +990,8 @@ func (s *Store) SearchMessages(listID int64, query string, limit, offset int) ([
return out, total, err
}
// ThreadMessages returns every message of a thread, oldest first, with bodies.
// ThreadMessages returns every message of a thread, oldest first, without bodies: a thread listing
// shows subjects, and a thread can hold many messages of up to 1 MiB of text each.
func (s *Store) ThreadMessages(listID, threadID int64) ([]Message, error) {
rows, err := s.db.Query(`SELECT `+msgCols+`, body FROM archive WHERE list_id=? AND thread_id=? ORDER BY date, id`, listID, threadID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Message
for rows.Next() {
m, err := scanMessage(rows, true)
if err != nil {
return nil, err
}
out = append(out, m)
}
return out, rows.Err()
return s.queryMessages(`SELECT `+msgCols+` FROM archive WHERE list_id=? AND thread_id=? ORDER BY date, id`, listID, threadID)
}