From 4ac967e00179230701e840394a89db443779d4cd Mon Sep 17 00:00:00 2001 From: tgiachi Date: Mon, 9 Nov 2026 15:18:23 +0100 Subject: [PATCH] feat(archive): parse messages into archivable entries - Decode subject, sender name, date, Message-ID and In-Reply-To - Pick the plain text body from nested multipart, fall back to stripped HTML - Decode transfer encodings and charsets, repair invalid UTF-8, cap at 1 MiB - Synthesize a stable Message-ID when missing - Never expose a sender address domain; add email obfuscation for public pages --- go.mod | 1 + go.sum | 1 + internal/archive/archive.go | 179 ++++++++++++++++++++++++++++++ internal/archive/archive_test.go | 184 +++++++++++++++++++++++++++++++ 4 files changed, 365 insertions(+) create mode 100644 internal/archive/archive.go create mode 100644 internal/archive/archive_test.go diff --git a/go.mod b/go.mod index 42eda6d..162f38c 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/ncruces/go-strftime v1.1.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect golang.org/x/sys v0.49.0 // indirect + golang.org/x/text v0.14.0 // indirect modernc.org/libc v1.80.1 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.12.1 // indirect diff --git a/go.sum b/go.sum index 022429f..e916675 100644 --- a/go.sum +++ b/go.sum @@ -49,6 +49,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= diff --git a/internal/archive/archive.go b/internal/archive/archive.go new file mode 100644 index 0000000..afc76d9 --- /dev/null +++ b/internal/archive/archive.go @@ -0,0 +1,179 @@ +// Package archive extracts what the archive needs from a raw email message. +package archive + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "html" + "io" + "regexp" + "strings" + "time" + "unicode/utf8" + + "github.com/emersion/go-message" + "github.com/emersion/go-message/mail" +) + +// Register charset decoders (iso-8859-*, windows-125x, ...). +import _ "github.com/emersion/go-message/charset" + +// MaxBody is the largest body stored for one message, in bytes. +const MaxBody = 1 << 20 + +// Entry is the archivable content of a message. +type Entry struct { + MessageID string + InReplyTo string + Subject string + Sender string // display name only, never the address domain + Date time.Time + Body string // plain text, UTF-8 +} + +var ( + msgIDRe = regexp.MustCompile(`<[^<>\s]+>`) + scriptRe = regexp.MustCompile(`(?is)]*>.*?`) + styleRe = regexp.MustCompile(`(?is)]*>.*?`) + breakRe = regexp.MustCompile(`(?i)|`) + tagRe = regexp.MustCompile(`<[^>]*>`) + blankRe = regexp.MustCompile(`\n{3,}`) + trailingRe = regexp.MustCompile(`[ \t]+\n`) + emailRe = regexp.MustCompile(`([A-Za-z0-9._%+\-]+)@[A-Za-z0-9\-]+(?:\.[A-Za-z0-9\-]+)*\.[A-Za-z]{2,}`) +) + +// Parse reads an RFC 5322 message. now is used when the Date header is missing +// or invalid. +func Parse(raw []byte, now time.Time) (Entry, error) { + ent, err := message.Read(bytes.NewReader(raw)) + if err != nil && !message.IsUnknownCharset(err) { + return Entry{}, fmt.Errorf("parse message: %w", err) + } + if ent == nil || ent.Header.Len() == 0 { + return Entry{}, errors.New("parse message: no headers") + } + h := mail.Header{Header: ent.Header} + + e := Entry{Date: now} + if d, err := h.Date(); err == nil && !d.IsZero() { + e.Date = d + } + if e.Subject, _ = h.Subject(); strings.TrimSpace(e.Subject) == "" { + e.Subject = "(no subject)" + } + e.Subject = clean(strings.Join(strings.Fields(e.Subject), " ")) + e.Sender = senderName(h) + + if id := msgIDRe.FindString(ent.Header.Get("Message-Id")); id != "" { + e.MessageID = id + } else { + sum := sha256.Sum256(raw) + e.MessageID = "<" + hex.EncodeToString(sum[:12]) + "@parsimony.invalid>" + } + if id := msgIDRe.FindString(ent.Header.Get("In-Reply-To")); id != "" { + e.InReplyTo = id + } else if refs := msgIDRe.FindAllString(ent.Header.Get("References"), -1); len(refs) > 0 { + e.InReplyTo = refs[len(refs)-1] + } + + plain, htmlBody := bodies(ent) + if strings.TrimSpace(plain) == "" && htmlBody != "" { + plain = stripHTML(htmlBody) + } + e.Body = clean(plain) + return e, nil +} + +func senderName(h mail.Header) string { + list, err := h.AddressList("From") + if err != nil || len(list) == 0 { + return "" + } + if n := strings.TrimSpace(list[0].Name); n != "" { + return clean(n) + } + local, _, _ := strings.Cut(list[0].Address, "@") + return clean(local) +} + +// bodies returns the first plain text and the first HTML body found. +func bodies(e *message.Entity) (plain, htmlBody string) { + ct, _, _ := e.Header.ContentType() + if ct == "" { + ct = "text/plain" + } + if strings.HasPrefix(ct, "multipart/") { + mr := e.MultipartReader() + if mr == nil { + return "", "" + } + for { + part, err := mr.NextPart() + if err == io.EOF { + break + } + if err != nil && !message.IsUnknownCharset(err) { + break + } + if disp, _, _ := part.Header.ContentDisposition(); disp == "attachment" { + continue + } + p, h := bodies(part) + if plain == "" { + plain = p + } + if htmlBody == "" { + htmlBody = h + } + } + return plain, htmlBody + } + switch ct { + case "text/plain": + plain = readLimited(e.Body) + case "text/html": + htmlBody = readLimited(e.Body) + } + return plain, htmlBody +} + +func readLimited(r io.Reader) string { + b, _ := io.ReadAll(io.LimitReader(r, MaxBody+4)) + return string(b) +} + +func stripHTML(s string) string { + s = scriptRe.ReplaceAllString(s, "") + s = styleRe.ReplaceAllString(s, "") + s = breakRe.ReplaceAllString(s, "\n") + s = tagRe.ReplaceAllString(s, "") + return html.UnescapeString(s) +} + +// clean makes text safe to store: valid UTF-8, no NUL, LF line ends, capped. +func clean(s string) string { + s = strings.ToValidUTF8(s, "\uFFFD") + s = strings.ReplaceAll(s, "\x00", "") + s = strings.ReplaceAll(s, "\r\n", "\n") + s = strings.ReplaceAll(s, "\r", "\n") + if len(s) > MaxBody { + s = s[:MaxBody] + for !utf8.ValidString(s) { // do not cut a character in half + s = s[:len(s)-1] + } + } + if strings.Contains(s, "\n\n\n") || strings.Contains(s, " \n") { + s = trailingRe.ReplaceAllString(s, "\n") + s = blankRe.ReplaceAllString(s, "\n\n") + } + return s +} + +// Obfuscate hides the domain of email addresses so a public archive cannot be +// harvested for them: alice@example.com becomes alice@…. +func Obfuscate(s string) string { + return emailRe.ReplaceAllString(s, "${1}@…") +} diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go new file mode 100644 index 0000000..a2f14e6 --- /dev/null +++ b/internal/archive/archive_test.go @@ -0,0 +1,184 @@ +package archive + +import ( + "strings" + "testing" + "time" +) + +var now = time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC) + +func parse(t *testing.T, raw string) Entry { + t.Helper() + e, err := Parse([]byte(raw), now) + if err != nil { + t.Fatalf("parse: %v", err) + } + return e +} + +func TestPlainMessage(t *testing.T) { + e := parse(t, "From: Alice Smith \r\nSubject: Hello\r\nMessage-ID: \r\n"+ + "Date: Mon, 02 Jan 2006 15:04:05 +0000\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nline one\r\nline two\r\n") + if e.Subject != "Hello" || e.Sender != "Alice Smith" || e.MessageID != "" { + t.Fatalf("entry: %+v", e) + } + if e.Body != "line one\nline two\n" { + t.Fatalf("body = %q", e.Body) + } + if !e.Date.Equal(time.Date(2006, 1, 2, 15, 4, 5, 0, time.UTC)) { + t.Fatalf("date = %v", e.Date) + } +} + +func TestSenderWithoutNameNeverExposesTheDomain(t *testing.T) { + e := parse(t, "From: bob@secret.example.com\r\nSubject: x\r\n\r\nhi\r\n") + if e.Sender != "bob" { + t.Fatalf("sender = %q", e.Sender) + } +} + +func TestEncodedSubjectAndSender(t *testing.T) { + e := parse(t, "From: =?UTF-8?B?w4lsw6g=?= \r\nSubject: =?ISO-8859-1?Q?Caf=E9_ol=E9?=\r\n\r\nx\r\n") + if e.Subject != "Café olé" || e.Sender != "Él\u00e8" { + t.Fatalf("entry: %q %q", e.Subject, e.Sender) + } +} + +func TestBase64UTF8Body(t *testing.T) { + e := parse(t, "Subject: b\r\nContent-Type: text/plain; charset=utf-8\r\nContent-Transfer-Encoding: base64\r\n\r\n"+ + "Y2lhbyBtb25kbyDimIM=\r\n") + if strings.TrimSpace(e.Body) != "ciao mondo ☃" { + t.Fatalf("body = %q", e.Body) + } +} + +func TestQuotedPrintableLatin1Body(t *testing.T) { + e := parse(t, "Subject: q\r\nContent-Type: text/plain; charset=iso-8859-1\r\nContent-Transfer-Encoding: quoted-printable\r\n\r\n"+ + "caff=E8 l=E0\r\n") + if strings.TrimSpace(e.Body) != "caffè là" { + t.Fatalf("body = %q", e.Body) + } +} + +func TestMultipartAlternativePrefersPlain(t *testing.T) { + e := parse(t, "Subject: m\r\nContent-Type: multipart/alternative; boundary=BB\r\n\r\n"+ + "--BB\r\nContent-Type: text/html\r\n\r\n

html version

\r\n"+ + "--BB\r\nContent-Type: text/plain\r\n\r\nplain version\r\n--BB--\r\n") + if strings.TrimSpace(e.Body) != "plain version" { + t.Fatalf("body = %q", e.Body) + } +} + +func TestMultipartMixedWithAttachmentAndNesting(t *testing.T) { + e := parse(t, "Subject: m\r\nContent-Type: multipart/mixed; boundary=OUT\r\n\r\n"+ + "--OUT\r\nContent-Type: multipart/alternative; boundary=IN\r\n\r\n"+ + "--IN\r\nContent-Type: text/plain\r\n\r\nreal text\r\n--IN--\r\n"+ + "--OUT\r\nContent-Type: application/pdf\r\nContent-Disposition: attachment; filename=a.pdf\r\nContent-Transfer-Encoding: base64\r\n\r\nJVBERi0=\r\n--OUT--\r\n") + if strings.TrimSpace(e.Body) != "real text" { + t.Fatalf("body = %q", e.Body) + } + if strings.Contains(e.Body, "JVBER") { + t.Fatal("attachments must not leak into the body") + } +} + +func TestHTMLOnlyIsStripped(t *testing.T) { + e := parse(t, "Subject: h\r\nContent-Type: text/html\r\n\r\n"+ + "

Hello & welcome

second

\r\n") + if strings.ContainsAny(e.Body, "<>") || strings.Contains(e.Body, "alert") || strings.Contains(e.Body, "color") { + t.Fatalf("html not stripped: %q", e.Body) + } + if !strings.Contains(e.Body, "Hello & welcome") || !strings.Contains(e.Body, "second") { + t.Fatalf("text lost: %q", e.Body) + } +} + +func TestUnknownCharsetStillArchives(t *testing.T) { + e := parse(t, "Subject: u\r\nContent-Type: text/plain; charset=x-made-up\r\n\r\nstill here\r\n") + if !strings.Contains(e.Body, "still here") { + t.Fatalf("body = %q", e.Body) + } +} + +func TestMissingMessageIDIsSynthesizedAndStable(t *testing.T) { + raw := "From: a@x.com\r\nSubject: no id\r\n\r\nbody\r\n" + a, b := parse(t, raw), parse(t, raw) + if a.MessageID == "" || !strings.HasPrefix(a.MessageID, "<") || a.MessageID != b.MessageID { + t.Fatalf("ids: %q %q", a.MessageID, b.MessageID) + } + if c := parse(t, raw+"x"); c.MessageID == a.MessageID { + t.Fatal("different messages need different ids") + } +} + +func TestInReplyToAndReferences(t *testing.T) { + e := parse(t, "Subject: r\r\nIn-Reply-To: (comment)\r\n\r\nx\r\n") + if e.InReplyTo != "" { + t.Fatalf("in-reply-to = %q", e.InReplyTo) + } + e = parse(t, "Subject: r\r\nReferences: \r\n\r\nx\r\n") + if e.InReplyTo != "" { + t.Fatalf("references fallback = %q", e.InReplyTo) + } + if e = parse(t, "Subject: r\r\n\r\nx\r\n"); e.InReplyTo != "" { + t.Fatalf("none expected, got %q", e.InReplyTo) + } +} + +func TestBadOrMissingDateFallsBackToNow(t *testing.T) { + for _, h := range []string{"", "Date: not a date\r\n"} { + if e := parse(t, h+"Subject: d\r\n\r\nx\r\n"); !e.Date.Equal(now) { + t.Fatalf("%q: date = %v", h, e.Date) + } + } +} + +func TestMissingSubject(t *testing.T) { + if e := parse(t, "From: a@x.com\r\n\r\nx\r\n"); e.Subject != "(no subject)" { + t.Fatalf("subject = %q", e.Subject) + } +} + +func TestBodyIsCappedAndCleaned(t *testing.T) { + big := strings.Repeat("a", MaxBody+5000) + e := parse(t, "Subject: big\r\n\r\n"+big+"\r\n") + if len(e.Body) > MaxBody { + t.Fatalf("body len %d exceeds cap", len(e.Body)) + } + e = parse(t, "Subject: nul\r\n\r\nab\x00cd\r\n") + if strings.Contains(e.Body, "\x00") { + t.Fatal("NUL must be removed") + } +} + +func TestInvalidUTF8IsRepaired(t *testing.T) { + e := parse(t, "Subject: bad\r\nContent-Type: text/plain\r\n\r\nok \xff\xfe end\r\n") + if !strings.Contains(e.Body, "ok") || !strings.Contains(e.Body, "end") { + t.Fatalf("body = %q", e.Body) + } + if strings.ToValidUTF8(e.Body, "") != e.Body { + t.Fatal("body must be valid UTF-8") + } +} + +func TestEmptyInputIsAnError(t *testing.T) { + if _, err := Parse(nil, now); err == nil { + t.Fatal("want error for an unparseable message") + } +} + +func TestObfuscate(t *testing.T) { + cases := map[string]string{ + "mail alice@example.com now": "mail alice@… now", + "a.b+c@sub.example.org, x@y.io": "a.b+c@…, x@…", + "no address here": "no address here", + "": "", + "twitter @handle only": "twitter @handle only", + } + for in, want := range cases { + if got := Obfuscate(in); got != want { + t.Errorf("Obfuscate(%q) = %q, want %q", in, got, want) + } + } +}