From d43c47ea08af858ffa741b66d6cc35aa6ba71eb2 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 11:48:51 +0100 Subject: [PATCH 1/4] fix(store): put the pragmas in the DSN so every connection has them - foreign_keys, busy_timeout(5000) and WAL now come from the DSN, not from an Exec on one connection - test: every connection of a 2-connection pool, in memory and on disk --- internal/store/store.go | 12 ++++++----- internal/store/store_test.go | 42 ++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/internal/store/store.go b/internal/store/store.go index 5abaedf..069b34e 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -215,16 +215,18 @@ CREATE TABLE IF NOT EXISTS settings ( // Open opens (and migrates) the database at path. Use ":memory:" for tests. func Open(path string) (*Store, error) { - db, err := sql.Open("sqlite", path) + // The pragmas are in the DSN so that every connection of the pool has them. + sep := "?" + if strings.Contains(path, "?") { + sep = "&" + } + dsn := path + sep + "_pragma=foreign_keys(1)&_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)" + db, err := sql.Open("sqlite", dsn) if err != nil { return nil, err } // An in-memory database is per-connection: keep a single one. db.SetMaxOpenConns(1) - if _, err := db.Exec("PRAGMA foreign_keys=ON; PRAGMA journal_mode=WAL;"); err != nil { - db.Close() - return nil, err - } if err := migrate(db); err != nil { db.Close() return nil, fmt.Errorf("migrate: %w", err) diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 3aa8483..50aab4e 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -1,7 +1,10 @@ package store import ( + "context" + "database/sql" "errors" + "path/filepath" "testing" ) @@ -124,3 +127,42 @@ func TestRemoveMember(t *testing.T) { t.Fatalf("want ErrNotFound, got %v", err) } } + +// Every connection of the pool must enforce foreign keys and wait on a busy +// database, not only the first one: the pragmas live in the DSN. +func TestEveryConnectionHasThePragmas(t *testing.T) { + for _, path := range []string{":memory:", filepath.Join(t.TempDir(), "p.db")} { + s, err := Open(path) + if err != nil { + t.Fatalf("open %s: %v", path, err) + } + s.db.SetMaxOpenConns(2) + ctx := context.Background() + c1, err := s.db.Conn(ctx) + if err != nil { + t.Fatal(err) + } + c2, err := s.db.Conn(ctx) + if err != nil { + t.Fatal(err) + } + for i, c := range []*sql.Conn{c1, c2} { + var fk, busy int + if err := c.QueryRowContext(ctx, "PRAGMA foreign_keys").Scan(&fk); err != nil || fk != 1 { + t.Errorf("%s conn %d: foreign_keys = %d, %v", path, i, fk, err) + } + if err := c.QueryRowContext(ctx, "PRAGMA busy_timeout").Scan(&busy); err != nil || busy != 5000 { + t.Errorf("%s conn %d: busy_timeout = %d, %v", path, i, busy, err) + } + if path != ":memory:" { + var mode string + if err := c.QueryRowContext(ctx, "PRAGMA journal_mode").Scan(&mode); err != nil || mode != "wal" { + t.Errorf("%s conn %d: journal_mode = %q, %v", path, i, mode, err) + } + } + } + c1.Close() + c2.Close() + s.Close() + } +} From 0a21e998e7cb62c706d8b4ecaf79d652bb2746f2 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Tue, 10 Nov 2026 11:51:13 +0100 Subject: [PATCH 2/4] fix(archive): bound the multipart walk and distrust extreme dates - walk at most 10 nested multipart levels and 100 parts, then use what was found: the post is still delivered, the archive stores less - read a text part only while it can still become the body, and stop at the first non-blank plain part, so memory stays near MaxBody - a Date more than 24 hours after the time of parsing becomes that time; a Date before 1990 is treated as missing - tests: 20000-level nesting finishes fast, exact limits, allocation bound, pathological HTML, date clamps --- internal/archive/archive.go | 67 +++++++---- internal/archive/archive_test.go | 197 +++++++++++++++++++++++++++++++ 2 files changed, 243 insertions(+), 21 deletions(-) diff --git a/internal/archive/archive.go b/internal/archive/archive.go index afc76d9..b8a8827 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -24,6 +24,19 @@ import _ "github.com/emersion/go-message/charset" // MaxBody is the largest body stored for one message, in bytes. const MaxBody = 1 << 20 +// Limits on how much of a message's structure is walked to find its body: a +// hostile message cannot make the parser recurse or loop for long. +const ( + maxDepth = 10 // levels of nested multipart + maxParts = 100 // parts visited in one message +) + +// Dates outside these bounds are not trusted: they would pin a message to the +// top (or the bottom) of the archive for good. +const maxFuture = 24 * time.Hour + +var minDate = time.Date(1990, 1, 1, 0, 0, 0, 0, time.UTC) + // Entry is the archivable content of a message. type Entry struct { MessageID string @@ -45,8 +58,8 @@ var ( emailRe = regexp.MustCompile(`([A-Za-z0-9._%+\-]+)@[A-Za-z0-9\-]+(?:\.[A-Za-z0-9\-]+)*\.[A-Za-z]{2,}`) ) -// Parse reads an RFC 5322 message. now is used when the Date header is missing -// or invalid. +// Parse reads an RFC 5322 message. now is used when the Date header is missing, +// invalid, before 1990, or more than 24 hours after now. func Parse(raw []byte, now time.Time) (Entry, error) { ent, err := message.Read(bytes.NewReader(raw)) if err != nil && !message.IsUnknownCharset(err) { @@ -58,7 +71,7 @@ func Parse(raw []byte, now time.Time) (Entry, error) { h := mail.Header{Header: ent.Header} e := Entry{Date: now} - if d, err := h.Date(); err == nil && !d.IsZero() { + if d, err := h.Date(); err == nil && !d.Before(minDate) && !d.After(now.Add(maxFuture)) { e.Date = d } if e.Subject, _ = h.Subject(); strings.TrimSpace(e.Subject) == "" { @@ -79,7 +92,9 @@ func Parse(raw []byte, now time.Time) (Entry, error) { e.InReplyTo = refs[len(refs)-1] } - plain, htmlBody := bodies(ent) + w := &walker{} + w.walk(ent, 0) + plain, htmlBody := w.plain, w.html if strings.TrimSpace(plain) == "" && htmlBody != "" { plain = stripHTML(htmlBody) } @@ -99,45 +114,55 @@ func senderName(h mail.Header) string { return clean(local) } -// bodies returns the first plain text and the first HTML body found. -func bodies(e *message.Entity) (plain, htmlBody string) { +// walker collects the first plain text and the first HTML body of a message. +type walker struct { + plain, html string + parts int +} + +// done reports whether the plain text is found: nothing else is needed. +func (w *walker) done() bool { return strings.TrimSpace(w.plain) != "" } + +func (w *walker) walk(e *message.Entity, depth int) { ct, _, _ := e.Header.ContentType() if ct == "" { ct = "text/plain" } if strings.HasPrefix(ct, "multipart/") { + if depth >= maxDepth { + return + } mr := e.MultipartReader() if mr == nil { - return "", "" + return } - for { + for !w.done() && w.parts < maxParts { part, err := mr.NextPart() if err == io.EOF { - break + return } if err != nil && !message.IsUnknownCharset(err) { - break + return } + w.parts++ if disp, _, _ := part.Header.ContentDisposition(); disp == "attachment" { continue } - p, h := bodies(part) - if plain == "" { - plain = p - } - if htmlBody == "" { - htmlBody = h - } + w.walk(part, depth+1) } - return plain, htmlBody + return } + // Read a text part only if it can still become the body. switch ct { case "text/plain": - plain = readLimited(e.Body) + if w.plain == "" { + w.plain = readLimited(e.Body) + } case "text/html": - htmlBody = readLimited(e.Body) + if w.html == "" { + w.html = readLimited(e.Body) + } } - return plain, htmlBody } func readLimited(r io.Reader) string { diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index fa96e29..c2b50b2 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -1,6 +1,8 @@ package archive import ( + "fmt" + "runtime" "strings" "testing" "time" @@ -212,3 +214,198 @@ func TestCapNeverSplitsAMultibyteCharacter(t *testing.T) { t.Fatalf("body len %d", len(e.Body)) } } + +// nestedBomb is a message with a text part followed by a part that nests +// multipart/mixed levels deep. +func nestedBomb(levels int) string { + var b strings.Builder + b.WriteString("From: a@example.com\r\nSubject: bomb\r\nContent-Type: multipart/mixed; boundary=b0\r\n\r\n") + b.WriteString("--b0\r\nContent-Type: text/plain\r\n\r\nhello\r\n") + for i := 0; i < levels; i++ { + fmt.Fprintf(&b, "--b%d\r\nContent-Type: multipart/mixed; boundary=b%d\r\n\r\n", i, i+1) + } + b.WriteString("--b" + fmt.Sprint(levels) + "\r\nContent-Type: text/plain\r\n\r\ndeep\r\n") + return b.String() +} + +func parseWithin(t *testing.T, raw string, limit time.Duration) Entry { + t.Helper() + type result struct { + e Entry + err error + } + done := make(chan result, 1) + go func() { + e, err := Parse([]byte(raw), now) + done <- result{e, err} + }() + select { + case r := <-done: + if r.err != nil { + t.Fatalf("parse: %v", r.err) + } + return r.e + case <-time.After(limit): + t.Fatalf("Parse still running after %v", limit) + return Entry{} + } +} + +func TestDeeplyNestedMultipartIsBounded(t *testing.T) { + e := parseWithin(t, nestedBomb(20000), 2*time.Second) + if e.Body != "hello\n" && e.Body != "hello" { + t.Fatalf("body = %q", e.Body) + } +} + +func TestNestingDeeperThanTheLimitIsIgnored(t *testing.T) { + // text/plain at depth 3 is found, at depth 40 it is not. + shallow := "Content-Type: multipart/mixed; boundary=b0\r\n\r\n" + + "--b0\r\nContent-Type: multipart/mixed; boundary=b1\r\n\r\n" + + "--b1\r\nContent-Type: multipart/mixed; boundary=b2\r\n\r\n" + + "--b2\r\nContent-Type: text/plain\r\n\r\nfound\r\n" + if e := parse(t, "Subject: s\r\n"+shallow); !strings.Contains(e.Body, "found") { + t.Fatalf("shallow body = %q", e.Body) + } + if e := parse(t, strings.Replace(nestedBomb(40), "text/plain\r\n\r\nhello", "application/octet-stream\r\n\r\nhello", 1)); strings.Contains(e.Body, "deep") { + t.Fatalf("a part 40 levels deep was read: %q", e.Body) + } +} + +func TestManyPartsAreBounded(t *testing.T) { + var b strings.Builder + b.WriteString("Subject: many\r\nContent-Type: multipart/mixed; boundary=b\r\n\r\n") + for i := 0; i < 5000; i++ { + b.WriteString("--b\r\nContent-Type: application/octet-stream\r\n\r\nx\r\n") + } + b.WriteString("--b\r\nContent-Type: text/plain\r\n\r\nlast\r\n") + e := parseWithin(t, b.String(), 2*time.Second) + if strings.Contains(e.Body, "last") { + t.Fatalf("part 5001 was read: %q", e.Body) + } + + // A text part among the first parts is still found. + b.Reset() + b.WriteString("Subject: many\r\nContent-Type: multipart/mixed; boundary=b\r\n\r\n") + for i := 0; i < 50; i++ { + b.WriteString("--b\r\nContent-Type: application/octet-stream\r\n\r\nx\r\n") + } + b.WriteString("--b\r\nContent-Type: text/plain\r\n\r\nearly\r\n") + if e := parse(t, b.String()); !strings.Contains(e.Body, "early") { + t.Fatalf("body = %q", e.Body) + } +} + +func TestOnlyTheTextPartsThatAreNeededAreRead(t *testing.T) { + // 60 text parts of 1 MiB each: the first wins and the rest are not read. + big := strings.Repeat("a", MaxBody) + var b strings.Builder + b.WriteString("Subject: big\r\nContent-Type: multipart/mixed; boundary=b\r\n\r\n") + for i := 0; i < 60; i++ { + b.WriteString("--b\r\nContent-Type: text/plain\r\n\r\n" + big + "\r\n") + } + raw := []byte(b.String()) + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + e, err := Parse(raw, now) + runtime.ReadMemStats(&after) + if err != nil { + t.Fatal(err) + } + if len(e.Body) != MaxBody { + t.Fatalf("body = %d bytes", len(e.Body)) + } + if used := after.TotalAlloc - before.TotalAlloc; used > 24<<20 { + t.Fatalf("Parse allocated %d MiB for a 1 MiB body", used>>20) + } +} + +func TestPathologicalHTMLIsNotQuadratic(t *testing.T) { + for name, frag := range map[string]string{ + "open script": "