diff --git a/internal/api/api.go b/internal/api/api.go index c0481ec..b40b9ab 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -2,13 +2,11 @@ package api import ( - "bytes" "context" "crypto/subtle" "encoding/json" "errors" "fmt" - "mime" "net/http" "net/mail" "strconv" @@ -353,18 +351,17 @@ type heldDTO struct { Size int `json:"size"` CreatedAt time.Time `json:"created_at"` Reason string `json:"reason"` // why the message was held + // The From header as received, and whether it names someone other than the envelope sender: + // what recipients will see as the author, against who really sent it. + From string `json:"from"` + Mismatch bool `json:"mismatch"` } -func subjectOf(data []byte) string { - m, err := mail.ReadMessage(bytes.NewReader(data)) - if err != nil { - return "" - } - subj := m.Header.Get("Subject") - if dec, err := new(mime.WordDecoder).DecodeHeader(subj); err == nil { - return dec - } - return subj +// fromMismatch reports whether the From header names an address other than the envelope sender, or +// no usable address at all. +func fromMismatch(fromHeader, sender string) bool { + a, err := mail.ParseAddress(fromHeader) + return err != nil || !strings.EqualFold(a.Address, sender) } func (s *server) listHeld(w http.ResponseWriter, r *http.Request) { @@ -383,7 +380,8 @@ func (s *server) listHeld(w http.ResponseWriter, r *http.Request) { } out := make([]heldDTO, 0, len(hs)) for _, h := range hs { - out = append(out, heldDTO{ID: h.ID, Sender: h.Sender, Subject: subjectOf(h.Data), Size: len(h.Data), CreatedAt: h.CreatedAt, Reason: h.Reason}) + out = append(out, heldDTO{ID: h.ID, Sender: h.Sender, Subject: h.Subject, Size: h.Size, CreatedAt: h.CreatedAt, Reason: h.Reason, + From: h.FromHeader, Mismatch: fromMismatch(h.FromHeader, h.Sender)}) } writeJSON(w, http.StatusOK, out) } diff --git a/internal/api/api_test.go b/internal/api/api_test.go index ff8d4bc..ac1e3ad 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -271,11 +271,13 @@ func heldServer(t *testing.T) (http.Handler, *store.Store, *fakeReleaser) { } type heldJSON struct { - ID int64 `json:"id"` - Sender string `json:"sender"` - Subject string `json:"subject"` - Size int `json:"size"` - Reason string `json:"reason"` + ID int64 `json:"id"` + Sender string `json:"sender"` + From string `json:"from"` + Mismatch bool `json:"mismatch"` + Subject string `json:"subject"` + Size int `json:"size"` + Reason string `json:"reason"` } func TestHeldQueueAPI(t *testing.T) { @@ -298,6 +300,35 @@ func TestHeldQueueAPI(t *testing.T) { expect(t, do(t, h, "DELETE", "/api/v1/held/abc", "", true), 404) } +func TestHeldListingShowsWhoReallyWrote(t *testing.T) { + s, _ := store.Open(":memory:") + defer s.Close() + l, _ := s.CreateList("dev", "example.org", "Dev") + s.HoldMessage(l.ID, "bob@x.com", []byte("From: Bob \r\nSubject: honest\r\n\r\nb"), "") + s.HoldMessage(l.ID, "bob@x.com", []byte("From: Eve \r\nSubject: passing as Eve\r\n\r\nb"), "") + s.HoldMessage(l.ID, "bob@x.com", []byte("Subject: no From at all\r\n\r\nb"), "") + s.HoldMessage(l.ID, "bob@x.com", []byte("From: not an address\r\nSubject: garbled\r\n\r\nb"), "") + h := New(s, token, nil) + + var hs []heldJSON + decode(t, do(t, h, "GET", "/api/v1/lists/1/held", "", true), &hs) + if len(hs) != 4 { + t.Fatalf("%+v", hs) + } + want := []struct { + from string + mismatch bool + }{{"Bob ", false}, {"Eve ", true}, {"", true}, {"not an address", true}} + for i, w := range want { + if hs[i].From != w.from || hs[i].Mismatch != w.mismatch { + t.Errorf("held %d: from %q mismatch %v, want %q %v", i+1, hs[i].From, hs[i].Mismatch, w.from, w.mismatch) + } + if hs[i].Sender != "bob@x.com" { + t.Errorf("the envelope sender stays visible: %q", hs[i].Sender) + } + } +} + func TestHeldRequiresAuth(t *testing.T) { h, _, _ := heldServer(t) expect(t, do(t, h, "POST", "/api/v1/held/1/approve", "", false), 401) diff --git a/internal/store/m2_test.go b/internal/store/m2_test.go index 7ac36c8..d1f03d0 100644 --- a/internal/store/m2_test.go +++ b/internal/store/m2_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "path/filepath" + "strings" "testing" "time" ) @@ -326,3 +327,143 @@ func TestOldHeldMessagesExpire(t *testing.T) { t.Fatalf("expiry is applied to the list being written to, not to others: %v", err) } } + +func TestHeldListingsCarryASummaryAndNeverTheBody(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + body := strings.Repeat("0123456789abcdef\r\n", 1000) + data := []byte("From: =?UTF-8?Q?Jos=C3=A9?= \r\nSubject: =?UTF-8?Q?Caff=C3=A8_a_las_nueve?=\r\n\r\n" + body) + id, err := s.HoldMessage(l.ID, "Alice@Example.com", data, "r") + if err != nil { + t.Fatal(err) + } + list, err := s.ListHeld(l.ID) + if err != nil || len(list) != 1 { + t.Fatalf("%v %v", list, err) + } + h := list[0] + if h.Data != nil { + t.Fatalf("a listing must not read the %d-byte body into memory", len(data)) + } + if h.Subject != "Caffè a las nueve" || h.FromHeader != "=?UTF-8?Q?Jos=C3=A9?= " || h.Size != len(data) || h.Sender != "alice@example.com" { + t.Fatalf("summary: %+v", h) + } + full, err := s.GetHeld(id) + if err != nil || len(full.Data) != len(data) || full.Subject != h.Subject { + t.Fatalf("a single message still comes with its body: %v %d", err, len(full.Data)) + } +} + +func TestHeldSummaryOfOddMessages(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + long := strings.Repeat("x", 500) + for name, tc := range map[string]struct { + data string + subject, from string + }{ + "no headers": {"\r\njust a body\r\n", "", ""}, + "empty": {"", "", ""}, + "no subject": {"From: a@x.com\r\n\r\nb", "", "a@x.com"}, + "long subject": {"Subject: " + long + "\r\n\r\nb", strings.Repeat("x", 200), ""}, + "long from": {"From: " + long + "\r\n\r\nb", "", strings.Repeat("x", 200)}, + "folded subject": {"Subject: one\r\n two\r\n\r\nb", "one two", ""}, + "control characters": {"Subject: a\x01b\x1b[31mred\r\n\r\nb", "ab[31mred", ""}, + } { + id, err := s.HoldMessage(l.ID, "a@x.com", []byte(tc.data), "") + if err != nil { + t.Fatalf("%s: %v", name, err) + } + h, _ := s.GetHeld(id) + if h.Subject != tc.subject || h.FromHeader != tc.from { + t.Errorf("%s: subject %q from %q, want %q %q", name, h.Subject, h.FromHeader, tc.subject, tc.from) + } + } +} + +func TestHeldQueueHasAByteCapAndAMessageCap(t *testing.T) { + oldMsg, oldList := heldMessageLimit, heldBytesLimit + heldMessageLimit, heldBytesLimit = 1000, 3500 + t.Cleanup(func() { heldMessageLimit, heldBytesLimit = oldMsg, oldList }) + s := newTestStore(t) + a, _ := s.CreateList("a", "example.org", "A") + b, _ := s.CreateList("b", "example.org", "B") + msg := func(n int) []byte { return []byte("Subject: s\r\n\r\n" + strings.Repeat("x", n-14)) } + + if _, err := s.HoldMessage(a.ID, "x@y.com", msg(1001), ""); !errors.Is(err, ErrHeldTooLarge) { + t.Fatalf("one message over the limit: want ErrHeldTooLarge, got %v", err) + } + if _, err := s.HoldMessage(a.ID, "x@y.com", msg(1000), ""); err != nil { + t.Fatalf("exactly at the limit is fine: %v", err) + } + var ids []int64 + for i := 0; i < 2; i++ { // 1000 + 1000 + 1000 = 3000 of 3500 + id, err := s.HoldMessage(a.ID, "x@y.com", msg(1000), "") + if err != nil { + t.Fatal(err) + } + ids = append(ids, id) + } + if _, err := s.HoldMessage(a.ID, "x@y.com", msg(600), ""); !errors.Is(err, ErrHeldFull) { + t.Fatalf("3000 + 600 > 3500: want ErrHeldFull, got %v", err) + } + if _, err := s.HoldMessage(a.ID, "x@y.com", msg(500), ""); err != nil { + t.Fatalf("3000 + 500 = 3500 fits: %v", err) + } + if _, err := s.HoldMessage(b.ID, "x@y.com", msg(1000), ""); err != nil { + t.Fatalf("another list has its own allowance: %v", err) + } + if err := s.DeleteHeld(ids[0]); err != nil { + t.Fatal(err) + } + if _, err := s.HoldMessage(a.ID, "x@y.com", msg(900), ""); err != nil { + t.Fatalf("moderating frees room: %v", err) + } +} + +func TestMigrationSummarisesExistingHeldMessages(t *testing.T) { + path := t.TempDir() + "/old.db" + s, _ := Open(path) + l, _ := s.CreateList("dev", "example.org", "Dev") + data := []byte("From: Bob \r\nSubject: Old held\r\n\r\nbody") + id, _ := s.HoldMessage(l.ID, "bob@x.com", data, "r") + s.db.Exec(`ALTER TABLE held DROP COLUMN subject`) + s.db.Exec(`ALTER TABLE held DROP COLUMN from_header`) + s.db.Exec(`ALTER TABLE held DROP COLUMN size`) + s.Close() + + s2, err := Open(path) + if err != nil { + t.Fatalf("migrate: %v", err) + } + defer s2.Close() + list, _ := s2.ListHeld(l.ID) + if len(list) != 1 || list[0].ID != id || list[0].Subject != "Old held" || list[0].FromHeader != "Bob " || list[0].Size != len(data) || list[0].Data != nil { + t.Fatalf("an existing held message is summarised by the migration: %+v", list) + } +} + +func TestThreadListingsDoNotLoadBodies(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + root, _, _ := s.ArchiveMessage(l.ID, msg("", "Root", strings.Repeat("big body ", 1000))) + reply := msg("", "Re: Root", "reply body") + reply.InReplyTo = "" + s.ArchiveMessage(l.ID, reply) + th, err := s.ThreadMessages(l.ID, root) + if err != nil || len(th) != 2 { + t.Fatalf("%v %v", th, err) + } + for _, m := range th { + if m.Body != "" { + t.Fatalf("a thread listing needs subjects, not bodies: %q", m.Body[:20]) + } + if m.Size == 0 || m.Subject == "" { + t.Fatalf("summary: %+v", m) + } + } + one, _ := s.GetMessage(root) + if !strings.HasPrefix(one.Body, "big body") { + t.Fatal("a single message still has its body") + } +} diff --git a/internal/store/store.go b/internal/store/store.go index cb27ebb..5abaedf 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -2,13 +2,17 @@ package store import ( + "bytes" "crypto/rand" "database/sql" "encoding/hex" "errors" "fmt" + "mime" + "net/mail" "strings" "time" + "unicode" _ "modernc.org/sqlite" ) @@ -18,6 +22,8 @@ var ( ErrNotFound = errors.New("not found") // ErrHeldFull is returned when a list's moderation queue has no more room. ErrHeldFull = errors.New("moderation queue is full") + // ErrHeldTooLarge is returned for a message too big to be held for moderation. + ErrHeldTooLarge = errors.New("message too large to hold for moderation") // ErrExists is returned when creating something that already exists. ErrExists = errors.New("already exists") ) @@ -43,6 +49,12 @@ const ( ActionLeave = "leave" ) +// Limits on the moderation queue of one list. They are variables so the tests can lower them. +var ( + heldMessageLimit = 10 << 20 // bytes in one held message + heldBytesLimit = 200 << 20 // bytes held for one list, all messages together +) + // BounceThreshold is the number of hard bounces after which a member is disabled. const BounceThreshold = 3 @@ -94,6 +106,11 @@ type Held struct { Data []byte CreatedAt time.Time Reason string // why the message was held, for the admin + + // What a listing needs, stored when the message is held so that a listing never has to read a body. + Subject string // decoded, at most 200 characters + FromHeader string // the From header as received, at most 200 characters + Size int // bytes of the message } // Pending is an unconfirmed subscribe or unsubscribe request. @@ -137,7 +154,10 @@ CREATE TABLE IF NOT EXISTS held ( sender TEXT NOT NULL, data BLOB NOT NULL, created_at INTEGER NOT NULL, - reason TEXT NOT NULL DEFAULT '' + reason TEXT NOT NULL DEFAULT '', + subject TEXT NOT NULL DEFAULT '', + from_header TEXT NOT NULL DEFAULT '', + size INTEGER NOT NULL DEFAULT 0 ); CREATE TABLE IF NOT EXISTS pending ( token TEXT PRIMARY KEY, @@ -275,11 +295,82 @@ func migrate(db *sql.DB) error { return err } } + if has, _ := hasColumn(db, "held", "size"); !has { + if err := addHeldSummary(db); err != nil { + return err + } + } } _, err := db.Exec(schema) return err } +// addHeldSummary adds the summary columns of the held queue and fills them for the messages +// already there, one message at a time so that a big queue is not loaded into memory. +func addHeldSummary(db *sql.DB) error { + for _, stmt := range []string{ + `ALTER TABLE held ADD COLUMN subject TEXT NOT NULL DEFAULT ''`, + `ALTER TABLE held ADD COLUMN from_header TEXT NOT NULL DEFAULT ''`, + `ALTER TABLE held ADD COLUMN size INTEGER NOT NULL DEFAULT 0`, + `UPDATE held SET size=length(data)`, + } { + if _, err := db.Exec(stmt); err != nil { + return err + } + } + rows, err := db.Query(`SELECT id FROM held`) + if err != nil { + return err + } + var ids []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + rows.Close() + return err + } + ids = append(ids, id) + } + rows.Close() + for _, id := range ids { + var data []byte + if err := db.QueryRow(`SELECT data FROM held WHERE id=?`, id).Scan(&data); err != nil { + return err + } + subject, from := summarize(data) + if _, err := db.Exec(`UPDATE held SET subject=?, from_header=? WHERE id=?`, subject, from, id); err != nil { + return err + } + } + return nil +} + +// summarize extracts what a moderation listing shows: the decoded subject and the From header, both +// without control characters and cut to 200 characters. +func summarize(data []byte) (subject, from string) { + m, err := mail.ReadMessage(bytes.NewReader(data)) + if err != nil { + return "", "" + } + clean := func(s string) string { + s = strings.Map(func(r rune) rune { + if unicode.IsControl(r) { + return -1 + } + return r + }, strings.Join(strings.Fields(s), " ")) + if r := []rune(s); len(r) > 200 { + s = string(r[:200]) + } + return s + } + raw := m.Header.Get("Subject") + if dec, err := new(mime.WordDecoder).DecodeHeader(raw); err == nil { + raw = dec + } + return clean(raw), clean(m.Header.Get("From")) +} + // Close closes the database. func (s *Store) Close() error { return s.db.Close() } @@ -468,34 +559,47 @@ func (s *Store) RecordHardBounce(memberID int64, threshold int) (bool, error) { // Held messages. -// HoldMessage queues a message for moderation and returns its id. reason tells the admin why. +// HoldMessage queues a message for moderation and returns its id. reason tells the admin why. A list +// holds a bounded amount: ErrHeldTooLarge for one message over heldMessageLimit, ErrHeldFull when the +// number of messages or the bytes held for the list would be exceeded. func (s *Store) HoldMessage(listID int64, sender string, data []byte, reason string) (int64, error) { if data == nil { data = []byte{} // the column is NOT NULL } + if len(data) > heldMessageLimit { + return 0, ErrHeldTooLarge + } now := s.Now() if _, err := s.db.Exec(`DELETE FROM held WHERE list_id=? AND created_at= maxHeldPerList { + if n >= maxHeldPerList || bytesHeld+len(data) > heldBytesLimit { return 0, ErrHeldFull } - res, err := s.db.Exec(`INSERT INTO held(list_id, sender, data, created_at, reason) VALUES(?,?,?,?,?)`, - listID, norm(sender), data, now.Unix(), reason) + subject, from := summarize(data) + res, err := s.db.Exec(`INSERT INTO held(list_id, sender, data, created_at, reason, subject, from_header, size) VALUES(?,?,?,?,?,?,?,?)`, + listID, norm(sender), data, now.Unix(), reason, subject, from, len(data)) if err != nil { return 0, err } return res.LastInsertId() } -func scanHeld(row interface{ Scan(...any) error }) (Held, error) { +const heldCols = `id, list_id, sender, created_at, reason, subject, from_header, size` + +func scanHeld(row interface{ Scan(...any) error }, withData bool) (Held, error) { var h Held var ts int64 - err := row.Scan(&h.ID, &h.ListID, &h.Sender, &h.Data, &ts, &h.Reason) + var err error + if withData { + err = row.Scan(&h.ID, &h.ListID, &h.Sender, &ts, &h.Reason, &h.Subject, &h.FromHeader, &h.Size, &h.Data) + } else { + err = row.Scan(&h.ID, &h.ListID, &h.Sender, &ts, &h.Reason, &h.Subject, &h.FromHeader, &h.Size) + } if errors.Is(err, sql.ErrNoRows) { return Held{}, ErrNotFound } @@ -503,16 +607,17 @@ func scanHeld(row interface{ Scan(...any) error }) (Held, error) { return h, err } -// ListHeld returns the held messages of a list, oldest first. +// ListHeld returns the held messages of a list, oldest first. It carries the summary of each message +// and never its body (Data is nil): a listing must not read up to 25 MiB per message into memory. func (s *Store) ListHeld(listID int64) ([]Held, error) { - rows, err := s.db.Query(`SELECT id, list_id, sender, data, created_at, reason FROM held WHERE list_id=? ORDER BY id`, listID) + rows, err := s.db.Query(`SELECT `+heldCols+` FROM held WHERE list_id=? ORDER BY id`, listID) if err != nil { return nil, err } defer rows.Close() var out []Held for rows.Next() { - h, err := scanHeld(rows) + h, err := scanHeld(rows, false) if err != nil { return nil, err } @@ -521,9 +626,9 @@ func (s *Store) ListHeld(listID int64) ([]Held, error) { return out, rows.Err() } -// GetHeld returns a held message by id. +// GetHeld returns one held message with its body. func (s *Store) GetHeld(id int64) (Held, error) { - return scanHeld(s.db.QueryRow(`SELECT id, list_id, sender, data, created_at, reason FROM held WHERE id=?`, id)) + return scanHeld(s.db.QueryRow(`SELECT `+heldCols+`, data FROM held WHERE id=?`, id), true) } // DeleteHeld discards a held message. @@ -885,20 +990,8 @@ func (s *Store) SearchMessages(listID int64, query string, limit, offset int) ([ return out, total, err } -// ThreadMessages returns every message of a thread, oldest first, with bodies. +// ThreadMessages returns every message of a thread, oldest first, without bodies: a thread listing +// shows subjects, and a thread can hold many messages of up to 1 MiB of text each. func (s *Store) ThreadMessages(listID, threadID int64) ([]Message, error) { - rows, err := s.db.Query(`SELECT `+msgCols+`, body FROM archive WHERE list_id=? AND thread_id=? ORDER BY date, id`, listID, threadID) - if err != nil { - return nil, err - } - defer rows.Close() - var out []Message - for rows.Next() { - m, err := scanMessage(rows, true) - if err != nil { - return nil, err - } - out = append(out, m) - } - return out, rows.Err() + return s.queryMessages(`SELECT `+msgCols+` FROM archive WHERE list_id=? AND thread_id=? ORDER BY date, id`, listID, threadID) }