diff --git a/README.md b/README.md index 2bfe74a..ba46316 100644 --- a/README.md +++ b/README.md @@ -266,6 +266,12 @@ lists.example.org lmtp:inet:127.0.0.1:2424 full queue answers `452` (the sender retries later), a message over 10 MiB `552`. Messages nobody moderated for 30 days are dropped. Listings (API, CLI, UI) show a summary and never read the bodies. The held view shows the From header readers would see next to the real sender and flags a difference. +- **Hostile messages and slow clients.** The archive reads at most 10 levels of nested multipart and 100 parts per + message, then uses what it found (the post is still delivered). A `Date` header more than 24 hours ahead is replaced + by the arrival time, and one before 1990 is ignored, so a message cannot pin itself to the top of the archive. The + HTTP servers (API + admin UI, and the public web) cut a request that takes more than 30 s to read, a response that + takes more than 60 s to write, and a connection idle for 120 s; the archive import is the exception and may stream + for up to 30 minutes. - **API token.** `parsimony serve` refuses a token shorter than 24 characters or containing whitespace (`openssl rand -hex 24` makes a good one). After 10 wrong or missing tokens within 5 minutes an address is locked out for 5 minutes (HTTP 429, valid token included). The address is the TCP peer: behind a reverse proxy diff --git a/internal/api/api.go b/internal/api/api.go index b40b9ab..0ad23a7 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -561,6 +561,10 @@ func (s *server) discardQueued(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) } +// importTimeout is how long one archive import may read its body and write its +// summary, in place of the server's default read and write timeouts. +const importTimeout = 30 * time.Minute + // importArchive reads an mbox from the request body, streaming, and archives it. // Failed messages are reported in the summary; the request itself succeeds. func (s *server) importArchive(w http.ResponseWriter, r *http.Request) { @@ -572,6 +576,11 @@ func (s *server) importArchive(w http.ResponseWriter, r *http.Request) { storeError(w, err) return } + // A big mbox streams for longer than the server's default timeouts. Errors + // are ignored: a ResponseWriter that cannot extend deadlines has none. + rc := http.NewResponseController(w) + _ = rc.SetReadDeadline(time.Now().Add(importTimeout)) + _ = rc.SetWriteDeadline(time.Now().Add(importTimeout)) sum, err := archive.ImportMbox(s.store, id, r.Body, time.Now()) if err != nil { writeError(w, http.StatusBadRequest, fmt.Sprintf("import stopped after %d message(s): %v", sum.Imported, err)) diff --git a/internal/app/app.go b/internal/app/app.go index 8909328..270b888 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -29,6 +29,25 @@ import ( // nowFunc is the clock of the store (queue schedules, token expiry). Tests move it. var nowFunc = time.Now +// HTTP timeouts of the API + admin UI server and of the public web server, so +// a slow or silent client cannot pin a goroutine. The archive import extends +// its own deadlines (see api.importArchive). Variables so tests can shorten them. +var ( + httpReadTimeout = 30 * time.Second + httpWriteTimeout = 60 * time.Second + httpIdleTimeout = 120 * time.Second +) + +func newHTTPServer(h http.Handler) *http.Server { + return &http.Server{ + Handler: h, + ReadHeaderTimeout: 10 * time.Second, + ReadTimeout: httpReadTimeout, + WriteTimeout: httpWriteTimeout, + IdleTimeout: httpIdleTimeout, + } +} + // heloName is the EHLO name: the configured one, else this machine's host name. func heloName(configured string) string { if configured != "" { @@ -196,7 +215,7 @@ func Start(cfg config.Config, apiToken string) (*App, error) { mux := http.NewServeMux() mux.Handle("/api/", api.New(st, apiToken, releaser{rt})) mux.Handle("/", adminui.Embedded()) - a.web = &http.Server{Handler: mux, ReadHeaderTimeout: 10 * time.Second} + a.web = newHTTPServer(mux) } var pubLn net.Listener @@ -211,7 +230,7 @@ func Start(cfg config.Config, apiToken string) (*App, error) { } a.WebAddr = pubLn.Addr().String() a.lns = append(a.lns, pubLn) - a.pub = &http.Server{Handler: web.New(st, rt), ReadHeaderTimeout: 10 * time.Second} + a.pub = newHTTPServer(web.New(st, rt)) } var workerCtx context.Context diff --git a/internal/app/app_test.go b/internal/app/app_test.go index 832937c..a1f6cf3 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -1004,3 +1004,115 @@ func TestTheServerNameSettingReachesTheTLSCheck(t *testing.T) { t.Fatal("nothing may be sent to a relay whose certificate does not match") } } + +// shortTimeouts makes the HTTP timeouts small enough to test. +func shortTimeouts(t *testing.T) { + t.Helper() + r, w, i := httpReadTimeout, httpWriteTimeout, httpIdleTimeout + httpReadTimeout, httpWriteTimeout, httpIdleTimeout = 300*time.Millisecond, 300*time.Millisecond, 300*time.Millisecond + t.Cleanup(func() { httpReadTimeout, httpWriteTimeout, httpIdleTimeout = r, w, i }) +} + +// stalled sends a request that promises a body and never finishes it, and +// reports whether the server hung up within a few seconds. +func stalled(t *testing.T, addr, req string) bool { + t.Helper() + c, err := net.Dial("tcp", addr) + if err != nil { + t.Fatal(err) + } + defer c.Close() + if _, err := c.Write([]byte(req)); err != nil { + t.Fatal(err) + } + c.SetReadDeadline(time.Now().Add(4 * time.Second)) + _, err = io.Copy(io.Discard, c) // returns nil on EOF, an error on our deadline + return err == nil +} + +func TestSlowRequestsAreCutOnBothServers(t *testing.T) { + shortTimeouts(t) + relayAddr, _ := startSink(t) + a, err := Start(testConfig(t, relayAddr), "tok") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { a.Close() }) + hdr := "POST %s HTTP/1.1\r\nHost: x\r\nAuthorization: Bearer tok\r\nContent-Type: application/json\r\nContent-Length: 500\r\n\r\n{\"name\":" + for name, c := range map[string]struct{ addr, path string }{ + "api": {a.APIAddr, "/api/v1/lists"}, + "web": {a.WebAddr, "/unsubscribe/tok"}, + } { + if !stalled(t, c.addr, strings.Replace(hdr, "%s", c.path, 1)) { + t.Errorf("%s: a stalled body was not cut", name) + } + } + // An idle keep-alive connection is closed too. + for name, addr := range map[string]string{"api": a.APIAddr, "web": a.WebAddr} { + if !stalled(t, addr, "GET /nothing HTTP/1.1\r\nHost: x\r\n\r\n") { + t.Errorf("%s: an idle connection was kept", name) + } + } +} + +// slowReader hands out its data in two halves, a pause apart. +type slowReader struct { + parts [][]byte + pause time.Duration +} + +func (s *slowReader) Read(p []byte) (int, error) { + if len(s.parts) == 0 { + return 0, io.EOF + } + if len(s.parts) == 1 { + time.Sleep(s.pause) + } + n := copy(p, s.parts[0]) + s.parts = s.parts[1:] + return n, nil +} + +func TestArchiveImportIsNotCutByTheDefaultTimeouts(t *testing.T) { + shortTimeouts(t) + relayAddr, _ := startSink(t) + a, err := Start(testConfig(t, relayAddr), "tok") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { a.Close() }) + call(t, a, "POST", "/api/v1/lists", `{"name":"dev","domain":"example.org","title":"Dev"}`, "tok") + + m1 := "From a@example.com Tue Jan 2 15:04:05 2024\nFrom: a@example.com\nSubject: one\nMessage-ID: <1@x>\n\nbody one\n\n" + m2 := "From a@example.com Tue Jan 2 15:04:06 2024\nFrom: a@example.com\nSubject: two\nMessage-ID: <2@x>\n\nbody two\n" + body := &slowReader{parts: [][]byte{[]byte(m1), []byte(m2)}, pause: 3 * httpReadTimeout} + req, _ := http.NewRequest("POST", "http://"+a.APIAddr+"/api/v1/lists/1/archive/import", body) + req.Header.Set("Authorization", "Bearer tok") + req.ContentLength = -1 // chunked: the body is streamed + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("import: %v", err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + if resp.StatusCode != 200 || !strings.Contains(string(b), `"imported":2`) { + t.Fatalf("import = %d %s", resp.StatusCode, b) + } +} + +func TestHTTPServersCarryTheTimeouts(t *testing.T) { + relayAddr, _ := startSink(t) + a, err := Start(testConfig(t, relayAddr), "tok") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { a.Close() }) + if httpReadTimeout != 30*time.Second || httpWriteTimeout != 60*time.Second || httpIdleTimeout != 120*time.Second { + t.Fatalf("defaults: %v %v %v", httpReadTimeout, httpWriteTimeout, httpIdleTimeout) + } + for name, s := range map[string]*http.Server{"api": a.web, "web": a.pub} { + if s.ReadTimeout != httpReadTimeout || s.WriteTimeout != httpWriteTimeout || s.IdleTimeout != httpIdleTimeout || s.ReadHeaderTimeout == 0 { + t.Errorf("%s: timeouts %v %v %v %v", name, s.ReadHeaderTimeout, s.ReadTimeout, s.WriteTimeout, s.IdleTimeout) + } + } +} diff --git a/internal/archive/archive.go b/internal/archive/archive.go index afc76d9..b8a8827 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -24,6 +24,19 @@ import _ "github.com/emersion/go-message/charset" // MaxBody is the largest body stored for one message, in bytes. const MaxBody = 1 << 20 +// Limits on how much of a message's structure is walked to find its body: a +// hostile message cannot make the parser recurse or loop for long. +const ( + maxDepth = 10 // levels of nested multipart + maxParts = 100 // parts visited in one message +) + +// Dates outside these bounds are not trusted: they would pin a message to the +// top (or the bottom) of the archive for good. +const maxFuture = 24 * time.Hour + +var minDate = time.Date(1990, 1, 1, 0, 0, 0, 0, time.UTC) + // Entry is the archivable content of a message. type Entry struct { MessageID string @@ -45,8 +58,8 @@ var ( emailRe = regexp.MustCompile(`([A-Za-z0-9._%+\-]+)@[A-Za-z0-9\-]+(?:\.[A-Za-z0-9\-]+)*\.[A-Za-z]{2,}`) ) -// Parse reads an RFC 5322 message. now is used when the Date header is missing -// or invalid. +// Parse reads an RFC 5322 message. now is used when the Date header is missing, +// invalid, before 1990, or more than 24 hours after now. func Parse(raw []byte, now time.Time) (Entry, error) { ent, err := message.Read(bytes.NewReader(raw)) if err != nil && !message.IsUnknownCharset(err) { @@ -58,7 +71,7 @@ func Parse(raw []byte, now time.Time) (Entry, error) { h := mail.Header{Header: ent.Header} e := Entry{Date: now} - if d, err := h.Date(); err == nil && !d.IsZero() { + if d, err := h.Date(); err == nil && !d.Before(minDate) && !d.After(now.Add(maxFuture)) { e.Date = d } if e.Subject, _ = h.Subject(); strings.TrimSpace(e.Subject) == "" { @@ -79,7 +92,9 @@ func Parse(raw []byte, now time.Time) (Entry, error) { e.InReplyTo = refs[len(refs)-1] } - plain, htmlBody := bodies(ent) + w := &walker{} + w.walk(ent, 0) + plain, htmlBody := w.plain, w.html if strings.TrimSpace(plain) == "" && htmlBody != "" { plain = stripHTML(htmlBody) } @@ -99,45 +114,55 @@ func senderName(h mail.Header) string { return clean(local) } -// bodies returns the first plain text and the first HTML body found. -func bodies(e *message.Entity) (plain, htmlBody string) { +// walker collects the first plain text and the first HTML body of a message. +type walker struct { + plain, html string + parts int +} + +// done reports whether the plain text is found: nothing else is needed. +func (w *walker) done() bool { return strings.TrimSpace(w.plain) != "" } + +func (w *walker) walk(e *message.Entity, depth int) { ct, _, _ := e.Header.ContentType() if ct == "" { ct = "text/plain" } if strings.HasPrefix(ct, "multipart/") { + if depth >= maxDepth { + return + } mr := e.MultipartReader() if mr == nil { - return "", "" + return } - for { + for !w.done() && w.parts < maxParts { part, err := mr.NextPart() if err == io.EOF { - break + return } if err != nil && !message.IsUnknownCharset(err) { - break + return } + w.parts++ if disp, _, _ := part.Header.ContentDisposition(); disp == "attachment" { continue } - p, h := bodies(part) - if plain == "" { - plain = p - } - if htmlBody == "" { - htmlBody = h - } + w.walk(part, depth+1) } - return plain, htmlBody + return } + // Read a text part only if it can still become the body. switch ct { case "text/plain": - plain = readLimited(e.Body) + if w.plain == "" { + w.plain = readLimited(e.Body) + } case "text/html": - htmlBody = readLimited(e.Body) + if w.html == "" { + w.html = readLimited(e.Body) + } } - return plain, htmlBody } func readLimited(r io.Reader) string { diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index fa96e29..1add6bd 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -1,6 +1,8 @@ package archive import ( + "fmt" + "runtime" "strings" "testing" "time" @@ -212,3 +214,198 @@ func TestCapNeverSplitsAMultibyteCharacter(t *testing.T) { t.Fatalf("body len %d", len(e.Body)) } } + +// nestedBomb is a message with a text part followed by a part that nests +// multipart/mixed levels deep. +func nestedBomb(levels int) string { + var b strings.Builder + b.WriteString("From: a@example.com\r\nSubject: bomb\r\nContent-Type: multipart/mixed; boundary=b0\r\n\r\n") + b.WriteString("--b0\r\nContent-Type: text/plain\r\n\r\nhello\r\n") + for i := 0; i < levels; i++ { + fmt.Fprintf(&b, "--b%d\r\nContent-Type: multipart/mixed; boundary=b%d\r\n\r\n", i, i+1) + } + b.WriteString("--b" + fmt.Sprint(levels) + "\r\nContent-Type: text/plain\r\n\r\ndeep\r\n") + return b.String() +} + +func parseWithin(t *testing.T, raw string, limit time.Duration) Entry { + t.Helper() + type result struct { + e Entry + err error + } + done := make(chan result, 1) + go func() { + e, err := Parse([]byte(raw), now) + done <- result{e, err} + }() + select { + case r := <-done: + if r.err != nil { + t.Fatalf("parse: %v", r.err) + } + return r.e + case <-time.After(limit): + t.Fatalf("Parse still running after %v", limit) + return Entry{} + } +} + +func TestDeeplyNestedMultipartIsBounded(t *testing.T) { + e := parseWithin(t, nestedBomb(20000), 2*time.Second) + if e.Body != "hello\n" && e.Body != "hello" { + t.Fatalf("body = %q", e.Body) + } +} + +func TestNestingDeeperThanTheLimitIsIgnored(t *testing.T) { + // text/plain at depth 3 is found, at depth 40 it is not. + shallow := "Content-Type: multipart/mixed; boundary=b0\r\n\r\n" + + "--b0\r\nContent-Type: multipart/mixed; boundary=b1\r\n\r\n" + + "--b1\r\nContent-Type: multipart/mixed; boundary=b2\r\n\r\n" + + "--b2\r\nContent-Type: text/plain\r\n\r\nfound\r\n" + if e := parse(t, "Subject: s\r\n"+shallow); !strings.Contains(e.Body, "found") { + t.Fatalf("shallow body = %q", e.Body) + } + if e := parse(t, strings.Replace(nestedBomb(40), "text/plain\r\n\r\nhello", "application/octet-stream\r\n\r\nhello", 1)); strings.Contains(e.Body, "deep") { + t.Fatalf("a part 40 levels deep was read: %q", e.Body) + } +} + +func TestManyPartsAreBounded(t *testing.T) { + var b strings.Builder + b.WriteString("Subject: many\r\nContent-Type: multipart/mixed; boundary=b\r\n\r\n") + for i := 0; i < 5000; i++ { + b.WriteString("--b\r\nContent-Type: application/octet-stream\r\n\r\nx\r\n") + } + b.WriteString("--b\r\nContent-Type: text/plain\r\n\r\nlast\r\n") + e := parseWithin(t, b.String(), 2*time.Second) + if strings.Contains(e.Body, "last") { + t.Fatalf("part 5001 was read: %q", e.Body) + } + + // A text part among the first parts is still found. + b.Reset() + b.WriteString("Subject: many\r\nContent-Type: multipart/mixed; boundary=b\r\n\r\n") + for i := 0; i < 50; i++ { + b.WriteString("--b\r\nContent-Type: application/octet-stream\r\n\r\nx\r\n") + } + b.WriteString("--b\r\nContent-Type: text/plain\r\n\r\nearly\r\n") + if e := parse(t, b.String()); !strings.Contains(e.Body, "early") { + t.Fatalf("body = %q", e.Body) + } +} + +func TestOnlyTheTextPartsThatAreNeededAreRead(t *testing.T) { + // 30 text parts of 1 MiB each: the first wins and the rest are not read. + big := strings.Repeat("a", MaxBody) + var b strings.Builder + b.WriteString("Subject: big\r\nContent-Type: multipart/mixed; boundary=b\r\n\r\n") + for i := 0; i < 30; i++ { + b.WriteString("--b\r\nContent-Type: text/plain\r\n\r\n" + big + "\r\n") + } + raw := []byte(b.String()) + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + e, err := Parse(raw, now) + runtime.ReadMemStats(&after) + if err != nil { + t.Fatal(err) + } + if len(e.Body) != MaxBody { + t.Fatalf("body = %d bytes", len(e.Body)) + } + if used := after.TotalAlloc - before.TotalAlloc; used > 24<<20 { + t.Fatalf("Parse allocated %d MiB for a 1 MiB body", used>>20) + } +} + +func TestPathologicalHTMLIsNotQuadratic(t *testing.T) { + for name, frag := range map[string]string{ + "open script": "