- walk at most 10 nested multipart levels and 100 parts, then use what was found: the post is still delivered, the archive stores less - read a text part only while it can still become the body, and stop at the first non-blank plain part, so memory stays near MaxBody - a Date more than 24 hours after the time of parsing becomes that time; a Date before 1990 is treated as missing - tests: 20000-level nesting finishes fast, exact limits, allocation bound, pathological HTML, date clamps
204 lines
5.5 KiB
Go
204 lines
5.5 KiB
Go
// Package archive extracts what the archive needs from a raw email message.
|
|
package archive
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"html"
|
|
"io"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
"unicode/utf8"
|
|
|
|
"github.com/emersion/go-message"
|
|
"github.com/emersion/go-message/mail"
|
|
)
|
|
|
|
// Register charset decoders (iso-8859-*, windows-125x, ...).
|
|
import _ "github.com/emersion/go-message/charset"
|
|
|
|
// MaxBody is the largest body stored for one message, in bytes.
|
|
const MaxBody = 1 << 20
|
|
|
|
// Limits on how much of a message's structure is walked to find its body: a
|
|
// hostile message cannot make the parser recurse or loop for long.
|
|
const (
|
|
maxDepth = 10 // levels of nested multipart
|
|
maxParts = 100 // parts visited in one message
|
|
)
|
|
|
|
// Dates outside these bounds are not trusted: they would pin a message to the
|
|
// top (or the bottom) of the archive for good.
|
|
const maxFuture = 24 * time.Hour
|
|
|
|
var minDate = time.Date(1990, 1, 1, 0, 0, 0, 0, time.UTC)
|
|
|
|
// Entry is the archivable content of a message.
|
|
type Entry struct {
|
|
MessageID string
|
|
InReplyTo string
|
|
Subject string
|
|
Sender string // display name only, never the address domain
|
|
Date time.Time
|
|
Body string // plain text, UTF-8
|
|
}
|
|
|
|
var (
|
|
msgIDRe = regexp.MustCompile(`<[^<>\s]+>`)
|
|
scriptRe = regexp.MustCompile(`(?is)<script[^>]*>.*?</script\s*>`)
|
|
styleRe = regexp.MustCompile(`(?is)<style[^>]*>.*?</style\s*>`)
|
|
breakRe = regexp.MustCompile(`(?i)<br\s*/?>|</(p|div|li|tr|h[1-6]|blockquote)\s*>`)
|
|
tagRe = regexp.MustCompile(`<[^>]*>`)
|
|
blankRe = regexp.MustCompile(`\n{3,}`)
|
|
trailingRe = regexp.MustCompile(`[ \t]+\n`)
|
|
emailRe = regexp.MustCompile(`([A-Za-z0-9._%+\-]+)@[A-Za-z0-9\-]+(?:\.[A-Za-z0-9\-]+)*\.[A-Za-z]{2,}`)
|
|
)
|
|
|
|
// Parse reads an RFC 5322 message. now is used when the Date header is missing,
|
|
// invalid, before 1990, or more than 24 hours after now.
|
|
func Parse(raw []byte, now time.Time) (Entry, error) {
|
|
ent, err := message.Read(bytes.NewReader(raw))
|
|
if err != nil && !message.IsUnknownCharset(err) {
|
|
return Entry{}, fmt.Errorf("parse message: %w", err)
|
|
}
|
|
if ent == nil || ent.Header.Len() == 0 {
|
|
return Entry{}, errors.New("parse message: no headers")
|
|
}
|
|
h := mail.Header{Header: ent.Header}
|
|
|
|
e := Entry{Date: now}
|
|
if d, err := h.Date(); err == nil && !d.Before(minDate) && !d.After(now.Add(maxFuture)) {
|
|
e.Date = d
|
|
}
|
|
if e.Subject, _ = h.Subject(); strings.TrimSpace(e.Subject) == "" {
|
|
e.Subject = "(no subject)"
|
|
}
|
|
e.Subject = clean(strings.Join(strings.Fields(e.Subject), " "))
|
|
e.Sender = senderName(h)
|
|
|
|
if id := msgIDRe.FindString(ent.Header.Get("Message-Id")); id != "" {
|
|
e.MessageID = id
|
|
} else {
|
|
sum := sha256.Sum256(raw)
|
|
e.MessageID = "<" + hex.EncodeToString(sum[:12]) + "@parsimony.invalid>"
|
|
}
|
|
if id := msgIDRe.FindString(ent.Header.Get("In-Reply-To")); id != "" {
|
|
e.InReplyTo = id
|
|
} else if refs := msgIDRe.FindAllString(ent.Header.Get("References"), -1); len(refs) > 0 {
|
|
e.InReplyTo = refs[len(refs)-1]
|
|
}
|
|
|
|
w := &walker{}
|
|
w.walk(ent, 0)
|
|
plain, htmlBody := w.plain, w.html
|
|
if strings.TrimSpace(plain) == "" && htmlBody != "" {
|
|
plain = stripHTML(htmlBody)
|
|
}
|
|
e.Body = clean(plain)
|
|
return e, nil
|
|
}
|
|
|
|
func senderName(h mail.Header) string {
|
|
list, err := h.AddressList("From")
|
|
if err != nil || len(list) == 0 {
|
|
return ""
|
|
}
|
|
if n := strings.TrimSpace(list[0].Name); n != "" {
|
|
return clean(n)
|
|
}
|
|
local, _, _ := strings.Cut(list[0].Address, "@")
|
|
return clean(local)
|
|
}
|
|
|
|
// walker collects the first plain text and the first HTML body of a message.
|
|
type walker struct {
|
|
plain, html string
|
|
parts int
|
|
}
|
|
|
|
// done reports whether the plain text is found: nothing else is needed.
|
|
func (w *walker) done() bool { return strings.TrimSpace(w.plain) != "" }
|
|
|
|
func (w *walker) walk(e *message.Entity, depth int) {
|
|
ct, _, _ := e.Header.ContentType()
|
|
if ct == "" {
|
|
ct = "text/plain"
|
|
}
|
|
if strings.HasPrefix(ct, "multipart/") {
|
|
if depth >= maxDepth {
|
|
return
|
|
}
|
|
mr := e.MultipartReader()
|
|
if mr == nil {
|
|
return
|
|
}
|
|
for !w.done() && w.parts < maxParts {
|
|
part, err := mr.NextPart()
|
|
if err == io.EOF {
|
|
return
|
|
}
|
|
if err != nil && !message.IsUnknownCharset(err) {
|
|
return
|
|
}
|
|
w.parts++
|
|
if disp, _, _ := part.Header.ContentDisposition(); disp == "attachment" {
|
|
continue
|
|
}
|
|
w.walk(part, depth+1)
|
|
}
|
|
return
|
|
}
|
|
// Read a text part only if it can still become the body.
|
|
switch ct {
|
|
case "text/plain":
|
|
if w.plain == "" {
|
|
w.plain = readLimited(e.Body)
|
|
}
|
|
case "text/html":
|
|
if w.html == "" {
|
|
w.html = readLimited(e.Body)
|
|
}
|
|
}
|
|
}
|
|
|
|
func readLimited(r io.Reader) string {
|
|
b, _ := io.ReadAll(io.LimitReader(r, MaxBody+4))
|
|
return string(b)
|
|
}
|
|
|
|
func stripHTML(s string) string {
|
|
s = scriptRe.ReplaceAllString(s, "")
|
|
s = styleRe.ReplaceAllString(s, "")
|
|
s = breakRe.ReplaceAllString(s, "\n")
|
|
s = tagRe.ReplaceAllString(s, "")
|
|
return html.UnescapeString(s)
|
|
}
|
|
|
|
// clean makes text safe to store: valid UTF-8, no NUL, LF line ends, capped.
|
|
func clean(s string) string {
|
|
s = strings.ToValidUTF8(s, "\uFFFD")
|
|
s = strings.ReplaceAll(s, "\x00", "")
|
|
s = strings.ReplaceAll(s, "\r\n", "\n")
|
|
s = strings.ReplaceAll(s, "\r", "\n")
|
|
if len(s) > MaxBody {
|
|
s = s[:MaxBody]
|
|
for !utf8.ValidString(s) { // do not cut a character in half
|
|
s = s[:len(s)-1]
|
|
}
|
|
}
|
|
if strings.Contains(s, "\n\n\n") || strings.Contains(s, " \n") {
|
|
s = trailingRe.ReplaceAllString(s, "\n")
|
|
s = blankRe.ReplaceAllString(s, "\n\n")
|
|
}
|
|
return s
|
|
}
|
|
|
|
// Obfuscate hides the domain of email addresses so a public archive cannot be
|
|
// harvested for them: alice@example.com becomes alice@….
|
|
func Obfuscate(s string) string {
|
|
return emailRe.ReplaceAllString(s, "${1}@…")
|
|
}
|