parsimony/internal/archive/archive.go
tgiachi 0a21e998e7 fix(archive): bound the multipart walk and distrust extreme dates
- walk at most 10 nested multipart levels and 100 parts, then use what
  was found: the post is still delivered, the archive stores less
- read a text part only while it can still become the body, and stop at
  the first non-blank plain part, so memory stays near MaxBody
- a Date more than 24 hours after the time of parsing becomes that time;
  a Date before 1990 is treated as missing
- tests: 20000-level nesting finishes fast, exact limits, allocation
  bound, pathological HTML, date clamps
2026-11-10 11:51:13 +01:00

204 lines
5.5 KiB
Go

// Package archive extracts what the archive needs from a raw email message.
package archive
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"html"
"io"
"regexp"
"strings"
"time"
"unicode/utf8"
"github.com/emersion/go-message"
"github.com/emersion/go-message/mail"
)
// Register charset decoders (iso-8859-*, windows-125x, ...).
import _ "github.com/emersion/go-message/charset"
// MaxBody is the largest body stored for one message, in bytes.
const MaxBody = 1 << 20
// Limits on how much of a message's structure is walked to find its body: a
// hostile message cannot make the parser recurse or loop for long.
const (
maxDepth = 10 // levels of nested multipart
maxParts = 100 // parts visited in one message
)
// Dates outside these bounds are not trusted: they would pin a message to the
// top (or the bottom) of the archive for good.
const maxFuture = 24 * time.Hour
var minDate = time.Date(1990, 1, 1, 0, 0, 0, 0, time.UTC)
// Entry is the archivable content of a message.
type Entry struct {
MessageID string
InReplyTo string
Subject string
Sender string // display name only, never the address domain
Date time.Time
Body string // plain text, UTF-8
}
var (
msgIDRe = regexp.MustCompile(`<[^<>\s]+>`)
scriptRe = regexp.MustCompile(`(?is)<script[^>]*>.*?</script\s*>`)
styleRe = regexp.MustCompile(`(?is)<style[^>]*>.*?</style\s*>`)
breakRe = regexp.MustCompile(`(?i)<br\s*/?>|</(p|div|li|tr|h[1-6]|blockquote)\s*>`)
tagRe = regexp.MustCompile(`<[^>]*>`)
blankRe = regexp.MustCompile(`\n{3,}`)
trailingRe = regexp.MustCompile(`[ \t]+\n`)
emailRe = regexp.MustCompile(`([A-Za-z0-9._%+\-]+)@[A-Za-z0-9\-]+(?:\.[A-Za-z0-9\-]+)*\.[A-Za-z]{2,}`)
)
// Parse reads an RFC 5322 message. now is used when the Date header is missing,
// invalid, before 1990, or more than 24 hours after now.
func Parse(raw []byte, now time.Time) (Entry, error) {
ent, err := message.Read(bytes.NewReader(raw))
if err != nil && !message.IsUnknownCharset(err) {
return Entry{}, fmt.Errorf("parse message: %w", err)
}
if ent == nil || ent.Header.Len() == 0 {
return Entry{}, errors.New("parse message: no headers")
}
h := mail.Header{Header: ent.Header}
e := Entry{Date: now}
if d, err := h.Date(); err == nil && !d.Before(minDate) && !d.After(now.Add(maxFuture)) {
e.Date = d
}
if e.Subject, _ = h.Subject(); strings.TrimSpace(e.Subject) == "" {
e.Subject = "(no subject)"
}
e.Subject = clean(strings.Join(strings.Fields(e.Subject), " "))
e.Sender = senderName(h)
if id := msgIDRe.FindString(ent.Header.Get("Message-Id")); id != "" {
e.MessageID = id
} else {
sum := sha256.Sum256(raw)
e.MessageID = "<" + hex.EncodeToString(sum[:12]) + "@parsimony.invalid>"
}
if id := msgIDRe.FindString(ent.Header.Get("In-Reply-To")); id != "" {
e.InReplyTo = id
} else if refs := msgIDRe.FindAllString(ent.Header.Get("References"), -1); len(refs) > 0 {
e.InReplyTo = refs[len(refs)-1]
}
w := &walker{}
w.walk(ent, 0)
plain, htmlBody := w.plain, w.html
if strings.TrimSpace(plain) == "" && htmlBody != "" {
plain = stripHTML(htmlBody)
}
e.Body = clean(plain)
return e, nil
}
func senderName(h mail.Header) string {
list, err := h.AddressList("From")
if err != nil || len(list) == 0 {
return ""
}
if n := strings.TrimSpace(list[0].Name); n != "" {
return clean(n)
}
local, _, _ := strings.Cut(list[0].Address, "@")
return clean(local)
}
// walker collects the first plain text and the first HTML body of a message.
type walker struct {
plain, html string
parts int
}
// done reports whether the plain text is found: nothing else is needed.
func (w *walker) done() bool { return strings.TrimSpace(w.plain) != "" }
func (w *walker) walk(e *message.Entity, depth int) {
ct, _, _ := e.Header.ContentType()
if ct == "" {
ct = "text/plain"
}
if strings.HasPrefix(ct, "multipart/") {
if depth >= maxDepth {
return
}
mr := e.MultipartReader()
if mr == nil {
return
}
for !w.done() && w.parts < maxParts {
part, err := mr.NextPart()
if err == io.EOF {
return
}
if err != nil && !message.IsUnknownCharset(err) {
return
}
w.parts++
if disp, _, _ := part.Header.ContentDisposition(); disp == "attachment" {
continue
}
w.walk(part, depth+1)
}
return
}
// Read a text part only if it can still become the body.
switch ct {
case "text/plain":
if w.plain == "" {
w.plain = readLimited(e.Body)
}
case "text/html":
if w.html == "" {
w.html = readLimited(e.Body)
}
}
}
func readLimited(r io.Reader) string {
b, _ := io.ReadAll(io.LimitReader(r, MaxBody+4))
return string(b)
}
func stripHTML(s string) string {
s = scriptRe.ReplaceAllString(s, "")
s = styleRe.ReplaceAllString(s, "")
s = breakRe.ReplaceAllString(s, "\n")
s = tagRe.ReplaceAllString(s, "")
return html.UnescapeString(s)
}
// clean makes text safe to store: valid UTF-8, no NUL, LF line ends, capped.
func clean(s string) string {
s = strings.ToValidUTF8(s, "\uFFFD")
s = strings.ReplaceAll(s, "\x00", "")
s = strings.ReplaceAll(s, "\r\n", "\n")
s = strings.ReplaceAll(s, "\r", "\n")
if len(s) > MaxBody {
s = s[:MaxBody]
for !utf8.ValidString(s) { // do not cut a character in half
s = s[:len(s)-1]
}
}
if strings.Contains(s, "\n\n\n") || strings.Contains(s, " \n") {
s = trailingRe.ReplaceAllString(s, "\n")
s = blankRe.ReplaceAllString(s, "\n\n")
}
return s
}
// Obfuscate hides the domain of email addresses so a public archive cannot be
// harvested for them: alice@example.com becomes alice@….
func Obfuscate(s string) string {
return emailRe.ReplaceAllString(s, "${1}@…")
}