feat(archive): parse messages into archivable entries
- Decode subject, sender name, date, Message-ID and In-Reply-To - Pick the plain text body from nested multipart, fall back to stripped HTML - Decode transfer encodings and charsets, repair invalid UTF-8, cap at 1 MiB - Synthesize a stable Message-ID when missing - Never expose a sender address domain; add email obfuscation for public pages
This commit is contained in:
parent
d8275aaa1f
commit
4ac967e001
4 changed files with 365 additions and 0 deletions
1
go.mod
1
go.mod
|
|
@ -17,6 +17,7 @@ require (
|
|||
github.com/ncruces/go-strftime v1.1.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
golang.org/x/sys v0.49.0 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
modernc.org/libc v1.80.1 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/memory v1.12.1 // indirect
|
||||
|
|
|
|||
1
go.sum
1
go.sum
|
|
@ -49,6 +49,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
|||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
|
|
|
|||
179
internal/archive/archive.go
Normal file
179
internal/archive/archive.go
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
// Package archive extracts what the archive needs from a raw email message.
|
||||
package archive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"html"
|
||||
"io"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/emersion/go-message"
|
||||
"github.com/emersion/go-message/mail"
|
||||
)
|
||||
|
||||
// Register charset decoders (iso-8859-*, windows-125x, ...).
|
||||
import _ "github.com/emersion/go-message/charset"
|
||||
|
||||
// MaxBody is the largest body stored for one message, in bytes.
|
||||
const MaxBody = 1 << 20
|
||||
|
||||
// Entry is the archivable content of a message.
|
||||
type Entry struct {
|
||||
MessageID string
|
||||
InReplyTo string
|
||||
Subject string
|
||||
Sender string // display name only, never the address domain
|
||||
Date time.Time
|
||||
Body string // plain text, UTF-8
|
||||
}
|
||||
|
||||
var (
|
||||
msgIDRe = regexp.MustCompile(`<[^<>\s]+>`)
|
||||
scriptRe = regexp.MustCompile(`(?is)<script[^>]*>.*?</script\s*>`)
|
||||
styleRe = regexp.MustCompile(`(?is)<style[^>]*>.*?</style\s*>`)
|
||||
breakRe = regexp.MustCompile(`(?i)<br\s*/?>|</(p|div|li|tr|h[1-6]|blockquote)\s*>`)
|
||||
tagRe = regexp.MustCompile(`<[^>]*>`)
|
||||
blankRe = regexp.MustCompile(`\n{3,}`)
|
||||
trailingRe = regexp.MustCompile(`[ \t]+\n`)
|
||||
emailRe = regexp.MustCompile(`([A-Za-z0-9._%+\-]+)@[A-Za-z0-9\-]+(?:\.[A-Za-z0-9\-]+)*\.[A-Za-z]{2,}`)
|
||||
)
|
||||
|
||||
// Parse reads an RFC 5322 message. now is used when the Date header is missing
|
||||
// or invalid.
|
||||
func Parse(raw []byte, now time.Time) (Entry, error) {
|
||||
ent, err := message.Read(bytes.NewReader(raw))
|
||||
if err != nil && !message.IsUnknownCharset(err) {
|
||||
return Entry{}, fmt.Errorf("parse message: %w", err)
|
||||
}
|
||||
if ent == nil || ent.Header.Len() == 0 {
|
||||
return Entry{}, errors.New("parse message: no headers")
|
||||
}
|
||||
h := mail.Header{Header: ent.Header}
|
||||
|
||||
e := Entry{Date: now}
|
||||
if d, err := h.Date(); err == nil && !d.IsZero() {
|
||||
e.Date = d
|
||||
}
|
||||
if e.Subject, _ = h.Subject(); strings.TrimSpace(e.Subject) == "" {
|
||||
e.Subject = "(no subject)"
|
||||
}
|
||||
e.Subject = clean(strings.Join(strings.Fields(e.Subject), " "))
|
||||
e.Sender = senderName(h)
|
||||
|
||||
if id := msgIDRe.FindString(ent.Header.Get("Message-Id")); id != "" {
|
||||
e.MessageID = id
|
||||
} else {
|
||||
sum := sha256.Sum256(raw)
|
||||
e.MessageID = "<" + hex.EncodeToString(sum[:12]) + "@parsimony.invalid>"
|
||||
}
|
||||
if id := msgIDRe.FindString(ent.Header.Get("In-Reply-To")); id != "" {
|
||||
e.InReplyTo = id
|
||||
} else if refs := msgIDRe.FindAllString(ent.Header.Get("References"), -1); len(refs) > 0 {
|
||||
e.InReplyTo = refs[len(refs)-1]
|
||||
}
|
||||
|
||||
plain, htmlBody := bodies(ent)
|
||||
if strings.TrimSpace(plain) == "" && htmlBody != "" {
|
||||
plain = stripHTML(htmlBody)
|
||||
}
|
||||
e.Body = clean(plain)
|
||||
return e, nil
|
||||
}
|
||||
|
||||
func senderName(h mail.Header) string {
|
||||
list, err := h.AddressList("From")
|
||||
if err != nil || len(list) == 0 {
|
||||
return ""
|
||||
}
|
||||
if n := strings.TrimSpace(list[0].Name); n != "" {
|
||||
return clean(n)
|
||||
}
|
||||
local, _, _ := strings.Cut(list[0].Address, "@")
|
||||
return clean(local)
|
||||
}
|
||||
|
||||
// bodies returns the first plain text and the first HTML body found.
|
||||
func bodies(e *message.Entity) (plain, htmlBody string) {
|
||||
ct, _, _ := e.Header.ContentType()
|
||||
if ct == "" {
|
||||
ct = "text/plain"
|
||||
}
|
||||
if strings.HasPrefix(ct, "multipart/") {
|
||||
mr := e.MultipartReader()
|
||||
if mr == nil {
|
||||
return "", ""
|
||||
}
|
||||
for {
|
||||
part, err := mr.NextPart()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil && !message.IsUnknownCharset(err) {
|
||||
break
|
||||
}
|
||||
if disp, _, _ := part.Header.ContentDisposition(); disp == "attachment" {
|
||||
continue
|
||||
}
|
||||
p, h := bodies(part)
|
||||
if plain == "" {
|
||||
plain = p
|
||||
}
|
||||
if htmlBody == "" {
|
||||
htmlBody = h
|
||||
}
|
||||
}
|
||||
return plain, htmlBody
|
||||
}
|
||||
switch ct {
|
||||
case "text/plain":
|
||||
plain = readLimited(e.Body)
|
||||
case "text/html":
|
||||
htmlBody = readLimited(e.Body)
|
||||
}
|
||||
return plain, htmlBody
|
||||
}
|
||||
|
||||
func readLimited(r io.Reader) string {
|
||||
b, _ := io.ReadAll(io.LimitReader(r, MaxBody+4))
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func stripHTML(s string) string {
|
||||
s = scriptRe.ReplaceAllString(s, "")
|
||||
s = styleRe.ReplaceAllString(s, "")
|
||||
s = breakRe.ReplaceAllString(s, "\n")
|
||||
s = tagRe.ReplaceAllString(s, "")
|
||||
return html.UnescapeString(s)
|
||||
}
|
||||
|
||||
// clean makes text safe to store: valid UTF-8, no NUL, LF line ends, capped.
|
||||
func clean(s string) string {
|
||||
s = strings.ToValidUTF8(s, "\uFFFD")
|
||||
s = strings.ReplaceAll(s, "\x00", "")
|
||||
s = strings.ReplaceAll(s, "\r\n", "\n")
|
||||
s = strings.ReplaceAll(s, "\r", "\n")
|
||||
if len(s) > MaxBody {
|
||||
s = s[:MaxBody]
|
||||
for !utf8.ValidString(s) { // do not cut a character in half
|
||||
s = s[:len(s)-1]
|
||||
}
|
||||
}
|
||||
if strings.Contains(s, "\n\n\n") || strings.Contains(s, " \n") {
|
||||
s = trailingRe.ReplaceAllString(s, "\n")
|
||||
s = blankRe.ReplaceAllString(s, "\n\n")
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Obfuscate hides the domain of email addresses so a public archive cannot be
|
||||
// harvested for them: alice@example.com becomes alice@….
|
||||
func Obfuscate(s string) string {
|
||||
return emailRe.ReplaceAllString(s, "${1}@…")
|
||||
}
|
||||
184
internal/archive/archive_test.go
Normal file
184
internal/archive/archive_test.go
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
package archive
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
var now = time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||
|
||||
func parse(t *testing.T, raw string) Entry {
|
||||
t.Helper()
|
||||
e, err := Parse([]byte(raw), now)
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
func TestPlainMessage(t *testing.T) {
|
||||
e := parse(t, "From: Alice Smith <alice@example.com>\r\nSubject: Hello\r\nMessage-ID: <abc@x>\r\n"+
|
||||
"Date: Mon, 02 Jan 2006 15:04:05 +0000\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nline one\r\nline two\r\n")
|
||||
if e.Subject != "Hello" || e.Sender != "Alice Smith" || e.MessageID != "<abc@x>" {
|
||||
t.Fatalf("entry: %+v", e)
|
||||
}
|
||||
if e.Body != "line one\nline two\n" {
|
||||
t.Fatalf("body = %q", e.Body)
|
||||
}
|
||||
if !e.Date.Equal(time.Date(2006, 1, 2, 15, 4, 5, 0, time.UTC)) {
|
||||
t.Fatalf("date = %v", e.Date)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSenderWithoutNameNeverExposesTheDomain(t *testing.T) {
|
||||
e := parse(t, "From: bob@secret.example.com\r\nSubject: x\r\n\r\nhi\r\n")
|
||||
if e.Sender != "bob" {
|
||||
t.Fatalf("sender = %q", e.Sender)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodedSubjectAndSender(t *testing.T) {
|
||||
e := parse(t, "From: =?UTF-8?B?w4lsw6g=?= <e@x.com>\r\nSubject: =?ISO-8859-1?Q?Caf=E9_ol=E9?=\r\n\r\nx\r\n")
|
||||
if e.Subject != "Café olé" || e.Sender != "Él\u00e8" {
|
||||
t.Fatalf("entry: %q %q", e.Subject, e.Sender)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBase64UTF8Body(t *testing.T) {
|
||||
e := parse(t, "Subject: b\r\nContent-Type: text/plain; charset=utf-8\r\nContent-Transfer-Encoding: base64\r\n\r\n"+
|
||||
"Y2lhbyBtb25kbyDimIM=\r\n")
|
||||
if strings.TrimSpace(e.Body) != "ciao mondo ☃" {
|
||||
t.Fatalf("body = %q", e.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuotedPrintableLatin1Body(t *testing.T) {
|
||||
e := parse(t, "Subject: q\r\nContent-Type: text/plain; charset=iso-8859-1\r\nContent-Transfer-Encoding: quoted-printable\r\n\r\n"+
|
||||
"caff=E8 l=E0\r\n")
|
||||
if strings.TrimSpace(e.Body) != "caffè là" {
|
||||
t.Fatalf("body = %q", e.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartAlternativePrefersPlain(t *testing.T) {
|
||||
e := parse(t, "Subject: m\r\nContent-Type: multipart/alternative; boundary=BB\r\n\r\n"+
|
||||
"--BB\r\nContent-Type: text/html\r\n\r\n<p>html version</p>\r\n"+
|
||||
"--BB\r\nContent-Type: text/plain\r\n\r\nplain version\r\n--BB--\r\n")
|
||||
if strings.TrimSpace(e.Body) != "plain version" {
|
||||
t.Fatalf("body = %q", e.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartMixedWithAttachmentAndNesting(t *testing.T) {
|
||||
e := parse(t, "Subject: m\r\nContent-Type: multipart/mixed; boundary=OUT\r\n\r\n"+
|
||||
"--OUT\r\nContent-Type: multipart/alternative; boundary=IN\r\n\r\n"+
|
||||
"--IN\r\nContent-Type: text/plain\r\n\r\nreal text\r\n--IN--\r\n"+
|
||||
"--OUT\r\nContent-Type: application/pdf\r\nContent-Disposition: attachment; filename=a.pdf\r\nContent-Transfer-Encoding: base64\r\n\r\nJVBERi0=\r\n--OUT--\r\n")
|
||||
if strings.TrimSpace(e.Body) != "real text" {
|
||||
t.Fatalf("body = %q", e.Body)
|
||||
}
|
||||
if strings.Contains(e.Body, "JVBER") {
|
||||
t.Fatal("attachments must not leak into the body")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTMLOnlyIsStripped(t *testing.T) {
|
||||
e := parse(t, "Subject: h\r\nContent-Type: text/html\r\n\r\n"+
|
||||
"<html><head><style>p{color:red}</style></head><body><script>alert(1)</script><p>Hello & <b>welcome</b></p><p>second</p></body></html>\r\n")
|
||||
if strings.ContainsAny(e.Body, "<>") || strings.Contains(e.Body, "alert") || strings.Contains(e.Body, "color") {
|
||||
t.Fatalf("html not stripped: %q", e.Body)
|
||||
}
|
||||
if !strings.Contains(e.Body, "Hello & welcome") || !strings.Contains(e.Body, "second") {
|
||||
t.Fatalf("text lost: %q", e.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnknownCharsetStillArchives(t *testing.T) {
|
||||
e := parse(t, "Subject: u\r\nContent-Type: text/plain; charset=x-made-up\r\n\r\nstill here\r\n")
|
||||
if !strings.Contains(e.Body, "still here") {
|
||||
t.Fatalf("body = %q", e.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMissingMessageIDIsSynthesizedAndStable(t *testing.T) {
|
||||
raw := "From: a@x.com\r\nSubject: no id\r\n\r\nbody\r\n"
|
||||
a, b := parse(t, raw), parse(t, raw)
|
||||
if a.MessageID == "" || !strings.HasPrefix(a.MessageID, "<") || a.MessageID != b.MessageID {
|
||||
t.Fatalf("ids: %q %q", a.MessageID, b.MessageID)
|
||||
}
|
||||
if c := parse(t, raw+"x"); c.MessageID == a.MessageID {
|
||||
t.Fatal("different messages need different ids")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInReplyToAndReferences(t *testing.T) {
|
||||
e := parse(t, "Subject: r\r\nIn-Reply-To: <parent@x> (comment)\r\n\r\nx\r\n")
|
||||
if e.InReplyTo != "<parent@x>" {
|
||||
t.Fatalf("in-reply-to = %q", e.InReplyTo)
|
||||
}
|
||||
e = parse(t, "Subject: r\r\nReferences: <root@x> <mid@x> <last@x>\r\n\r\nx\r\n")
|
||||
if e.InReplyTo != "<last@x>" {
|
||||
t.Fatalf("references fallback = %q", e.InReplyTo)
|
||||
}
|
||||
if e = parse(t, "Subject: r\r\n\r\nx\r\n"); e.InReplyTo != "" {
|
||||
t.Fatalf("none expected, got %q", e.InReplyTo)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBadOrMissingDateFallsBackToNow(t *testing.T) {
|
||||
for _, h := range []string{"", "Date: not a date\r\n"} {
|
||||
if e := parse(t, h+"Subject: d\r\n\r\nx\r\n"); !e.Date.Equal(now) {
|
||||
t.Fatalf("%q: date = %v", h, e.Date)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMissingSubject(t *testing.T) {
|
||||
if e := parse(t, "From: a@x.com\r\n\r\nx\r\n"); e.Subject != "(no subject)" {
|
||||
t.Fatalf("subject = %q", e.Subject)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBodyIsCappedAndCleaned(t *testing.T) {
|
||||
big := strings.Repeat("a", MaxBody+5000)
|
||||
e := parse(t, "Subject: big\r\n\r\n"+big+"\r\n")
|
||||
if len(e.Body) > MaxBody {
|
||||
t.Fatalf("body len %d exceeds cap", len(e.Body))
|
||||
}
|
||||
e = parse(t, "Subject: nul\r\n\r\nab\x00cd\r\n")
|
||||
if strings.Contains(e.Body, "\x00") {
|
||||
t.Fatal("NUL must be removed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidUTF8IsRepaired(t *testing.T) {
|
||||
e := parse(t, "Subject: bad\r\nContent-Type: text/plain\r\n\r\nok \xff\xfe end\r\n")
|
||||
if !strings.Contains(e.Body, "ok") || !strings.Contains(e.Body, "end") {
|
||||
t.Fatalf("body = %q", e.Body)
|
||||
}
|
||||
if strings.ToValidUTF8(e.Body, "") != e.Body {
|
||||
t.Fatal("body must be valid UTF-8")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyInputIsAnError(t *testing.T) {
|
||||
if _, err := Parse(nil, now); err == nil {
|
||||
t.Fatal("want error for an unparseable message")
|
||||
}
|
||||
}
|
||||
|
||||
func TestObfuscate(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"mail alice@example.com now": "mail alice@… now",
|
||||
"a.b+c@sub.example.org, x@y.io": "a.b+c@…, x@…",
|
||||
"no address here": "no address here",
|
||||
"<bob@x.co>": "<bob@…>",
|
||||
"twitter @handle only": "twitter @handle only",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := Obfuscate(in); got != want {
|
||||
t.Errorf("Obfuscate(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue