diff --git a/go.mod b/go.mod
index 42eda6d..162f38c 100644
--- a/go.mod
+++ b/go.mod
@@ -17,6 +17,7 @@ require (
github.com/ncruces/go-strftime v1.1.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
golang.org/x/sys v0.49.0 // indirect
+ golang.org/x/text v0.14.0 // indirect
modernc.org/libc v1.80.1 // indirect
modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.12.1 // indirect
diff --git a/go.sum b/go.sum
index 022429f..e916675 100644
--- a/go.sum
+++ b/go.sum
@@ -49,6 +49,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
diff --git a/internal/archive/archive.go b/internal/archive/archive.go
new file mode 100644
index 0000000..afc76d9
--- /dev/null
+++ b/internal/archive/archive.go
@@ -0,0 +1,179 @@
+// Package archive extracts what the archive needs from a raw email message.
+package archive
+
+import (
+ "bytes"
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "html"
+ "io"
+ "regexp"
+ "strings"
+ "time"
+ "unicode/utf8"
+
+ "github.com/emersion/go-message"
+ "github.com/emersion/go-message/mail"
+)
+
+// Register charset decoders (iso-8859-*, windows-125x, ...).
+import _ "github.com/emersion/go-message/charset"
+
+// MaxBody is the largest body stored for one message, in bytes.
+const MaxBody = 1 << 20
+
+// Entry is the archivable content of a message.
+type Entry struct {
+ MessageID string
+ InReplyTo string
+ Subject string
+ Sender string // display name only, never the address domain
+ Date time.Time
+ Body string // plain text, UTF-8
+}
+
+var (
+ msgIDRe = regexp.MustCompile(`<[^<>\s]+>`)
+ scriptRe = regexp.MustCompile(`(?is)`)
+ styleRe = regexp.MustCompile(`(?is)`)
+ breakRe = regexp.MustCompile(`(?i)
|(p|div|li|tr|h[1-6]|blockquote)\s*>`)
+ tagRe = regexp.MustCompile(`<[^>]*>`)
+ blankRe = regexp.MustCompile(`\n{3,}`)
+ trailingRe = regexp.MustCompile(`[ \t]+\n`)
+ emailRe = regexp.MustCompile(`([A-Za-z0-9._%+\-]+)@[A-Za-z0-9\-]+(?:\.[A-Za-z0-9\-]+)*\.[A-Za-z]{2,}`)
+)
+
+// Parse reads an RFC 5322 message. now is used when the Date header is missing
+// or invalid.
+func Parse(raw []byte, now time.Time) (Entry, error) {
+ ent, err := message.Read(bytes.NewReader(raw))
+ if err != nil && !message.IsUnknownCharset(err) {
+ return Entry{}, fmt.Errorf("parse message: %w", err)
+ }
+ if ent == nil || ent.Header.Len() == 0 {
+ return Entry{}, errors.New("parse message: no headers")
+ }
+ h := mail.Header{Header: ent.Header}
+
+ e := Entry{Date: now}
+ if d, err := h.Date(); err == nil && !d.IsZero() {
+ e.Date = d
+ }
+ if e.Subject, _ = h.Subject(); strings.TrimSpace(e.Subject) == "" {
+ e.Subject = "(no subject)"
+ }
+ e.Subject = clean(strings.Join(strings.Fields(e.Subject), " "))
+ e.Sender = senderName(h)
+
+ if id := msgIDRe.FindString(ent.Header.Get("Message-Id")); id != "" {
+ e.MessageID = id
+ } else {
+ sum := sha256.Sum256(raw)
+ e.MessageID = "<" + hex.EncodeToString(sum[:12]) + "@parsimony.invalid>"
+ }
+ if id := msgIDRe.FindString(ent.Header.Get("In-Reply-To")); id != "" {
+ e.InReplyTo = id
+ } else if refs := msgIDRe.FindAllString(ent.Header.Get("References"), -1); len(refs) > 0 {
+ e.InReplyTo = refs[len(refs)-1]
+ }
+
+ plain, htmlBody := bodies(ent)
+ if strings.TrimSpace(plain) == "" && htmlBody != "" {
+ plain = stripHTML(htmlBody)
+ }
+ e.Body = clean(plain)
+ return e, nil
+}
+
+func senderName(h mail.Header) string {
+ list, err := h.AddressList("From")
+ if err != nil || len(list) == 0 {
+ return ""
+ }
+ if n := strings.TrimSpace(list[0].Name); n != "" {
+ return clean(n)
+ }
+ local, _, _ := strings.Cut(list[0].Address, "@")
+ return clean(local)
+}
+
+// bodies returns the first plain text and the first HTML body found.
+func bodies(e *message.Entity) (plain, htmlBody string) {
+ ct, _, _ := e.Header.ContentType()
+ if ct == "" {
+ ct = "text/plain"
+ }
+ if strings.HasPrefix(ct, "multipart/") {
+ mr := e.MultipartReader()
+ if mr == nil {
+ return "", ""
+ }
+ for {
+ part, err := mr.NextPart()
+ if err == io.EOF {
+ break
+ }
+ if err != nil && !message.IsUnknownCharset(err) {
+ break
+ }
+ if disp, _, _ := part.Header.ContentDisposition(); disp == "attachment" {
+ continue
+ }
+ p, h := bodies(part)
+ if plain == "" {
+ plain = p
+ }
+ if htmlBody == "" {
+ htmlBody = h
+ }
+ }
+ return plain, htmlBody
+ }
+ switch ct {
+ case "text/plain":
+ plain = readLimited(e.Body)
+ case "text/html":
+ htmlBody = readLimited(e.Body)
+ }
+ return plain, htmlBody
+}
+
+func readLimited(r io.Reader) string {
+ b, _ := io.ReadAll(io.LimitReader(r, MaxBody+4))
+ return string(b)
+}
+
+func stripHTML(s string) string {
+ s = scriptRe.ReplaceAllString(s, "")
+ s = styleRe.ReplaceAllString(s, "")
+ s = breakRe.ReplaceAllString(s, "\n")
+ s = tagRe.ReplaceAllString(s, "")
+ return html.UnescapeString(s)
+}
+
+// clean makes text safe to store: valid UTF-8, no NUL, LF line ends, capped.
+func clean(s string) string {
+ s = strings.ToValidUTF8(s, "\uFFFD")
+ s = strings.ReplaceAll(s, "\x00", "")
+ s = strings.ReplaceAll(s, "\r\n", "\n")
+ s = strings.ReplaceAll(s, "\r", "\n")
+ if len(s) > MaxBody {
+ s = s[:MaxBody]
+ for !utf8.ValidString(s) { // do not cut a character in half
+ s = s[:len(s)-1]
+ }
+ }
+ if strings.Contains(s, "\n\n\n") || strings.Contains(s, " \n") {
+ s = trailingRe.ReplaceAllString(s, "\n")
+ s = blankRe.ReplaceAllString(s, "\n\n")
+ }
+ return s
+}
+
+// Obfuscate hides the domain of email addresses so a public archive cannot be
+// harvested for them: alice@example.com becomes alice@….
+func Obfuscate(s string) string {
+ return emailRe.ReplaceAllString(s, "${1}@…")
+}
diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go
new file mode 100644
index 0000000..a2f14e6
--- /dev/null
+++ b/internal/archive/archive_test.go
@@ -0,0 +1,184 @@
+package archive
+
+import (
+ "strings"
+ "testing"
+ "time"
+)
+
+var now = time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)
+
+func parse(t *testing.T, raw string) Entry {
+ t.Helper()
+ e, err := Parse([]byte(raw), now)
+ if err != nil {
+ t.Fatalf("parse: %v", err)
+ }
+ return e
+}
+
+func TestPlainMessage(t *testing.T) {
+ e := parse(t, "From: Alice Smith \r\nSubject: Hello\r\nMessage-ID: \r\n"+
+ "Date: Mon, 02 Jan 2006 15:04:05 +0000\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nline one\r\nline two\r\n")
+ if e.Subject != "Hello" || e.Sender != "Alice Smith" || e.MessageID != "" {
+ t.Fatalf("entry: %+v", e)
+ }
+ if e.Body != "line one\nline two\n" {
+ t.Fatalf("body = %q", e.Body)
+ }
+ if !e.Date.Equal(time.Date(2006, 1, 2, 15, 4, 5, 0, time.UTC)) {
+ t.Fatalf("date = %v", e.Date)
+ }
+}
+
+func TestSenderWithoutNameNeverExposesTheDomain(t *testing.T) {
+ e := parse(t, "From: bob@secret.example.com\r\nSubject: x\r\n\r\nhi\r\n")
+ if e.Sender != "bob" {
+ t.Fatalf("sender = %q", e.Sender)
+ }
+}
+
+func TestEncodedSubjectAndSender(t *testing.T) {
+ e := parse(t, "From: =?UTF-8?B?w4lsw6g=?= \r\nSubject: =?ISO-8859-1?Q?Caf=E9_ol=E9?=\r\n\r\nx\r\n")
+ if e.Subject != "Café olé" || e.Sender != "Él\u00e8" {
+ t.Fatalf("entry: %q %q", e.Subject, e.Sender)
+ }
+}
+
+func TestBase64UTF8Body(t *testing.T) {
+ e := parse(t, "Subject: b\r\nContent-Type: text/plain; charset=utf-8\r\nContent-Transfer-Encoding: base64\r\n\r\n"+
+ "Y2lhbyBtb25kbyDimIM=\r\n")
+ if strings.TrimSpace(e.Body) != "ciao mondo ☃" {
+ t.Fatalf("body = %q", e.Body)
+ }
+}
+
+func TestQuotedPrintableLatin1Body(t *testing.T) {
+ e := parse(t, "Subject: q\r\nContent-Type: text/plain; charset=iso-8859-1\r\nContent-Transfer-Encoding: quoted-printable\r\n\r\n"+
+ "caff=E8 l=E0\r\n")
+ if strings.TrimSpace(e.Body) != "caffè là" {
+ t.Fatalf("body = %q", e.Body)
+ }
+}
+
+func TestMultipartAlternativePrefersPlain(t *testing.T) {
+ e := parse(t, "Subject: m\r\nContent-Type: multipart/alternative; boundary=BB\r\n\r\n"+
+ "--BB\r\nContent-Type: text/html\r\n\r\nhtml version
\r\n"+
+ "--BB\r\nContent-Type: text/plain\r\n\r\nplain version\r\n--BB--\r\n")
+ if strings.TrimSpace(e.Body) != "plain version" {
+ t.Fatalf("body = %q", e.Body)
+ }
+}
+
+func TestMultipartMixedWithAttachmentAndNesting(t *testing.T) {
+ e := parse(t, "Subject: m\r\nContent-Type: multipart/mixed; boundary=OUT\r\n\r\n"+
+ "--OUT\r\nContent-Type: multipart/alternative; boundary=IN\r\n\r\n"+
+ "--IN\r\nContent-Type: text/plain\r\n\r\nreal text\r\n--IN--\r\n"+
+ "--OUT\r\nContent-Type: application/pdf\r\nContent-Disposition: attachment; filename=a.pdf\r\nContent-Transfer-Encoding: base64\r\n\r\nJVBERi0=\r\n--OUT--\r\n")
+ if strings.TrimSpace(e.Body) != "real text" {
+ t.Fatalf("body = %q", e.Body)
+ }
+ if strings.Contains(e.Body, "JVBER") {
+ t.Fatal("attachments must not leak into the body")
+ }
+}
+
+func TestHTMLOnlyIsStripped(t *testing.T) {
+ e := parse(t, "Subject: h\r\nContent-Type: text/html\r\n\r\n"+
+ "Hello & welcome
second
\r\n")
+ if strings.ContainsAny(e.Body, "<>") || strings.Contains(e.Body, "alert") || strings.Contains(e.Body, "color") {
+ t.Fatalf("html not stripped: %q", e.Body)
+ }
+ if !strings.Contains(e.Body, "Hello & welcome") || !strings.Contains(e.Body, "second") {
+ t.Fatalf("text lost: %q", e.Body)
+ }
+}
+
+func TestUnknownCharsetStillArchives(t *testing.T) {
+ e := parse(t, "Subject: u\r\nContent-Type: text/plain; charset=x-made-up\r\n\r\nstill here\r\n")
+ if !strings.Contains(e.Body, "still here") {
+ t.Fatalf("body = %q", e.Body)
+ }
+}
+
+func TestMissingMessageIDIsSynthesizedAndStable(t *testing.T) {
+ raw := "From: a@x.com\r\nSubject: no id\r\n\r\nbody\r\n"
+ a, b := parse(t, raw), parse(t, raw)
+ if a.MessageID == "" || !strings.HasPrefix(a.MessageID, "<") || a.MessageID != b.MessageID {
+ t.Fatalf("ids: %q %q", a.MessageID, b.MessageID)
+ }
+ if c := parse(t, raw+"x"); c.MessageID == a.MessageID {
+ t.Fatal("different messages need different ids")
+ }
+}
+
+func TestInReplyToAndReferences(t *testing.T) {
+ e := parse(t, "Subject: r\r\nIn-Reply-To: (comment)\r\n\r\nx\r\n")
+ if e.InReplyTo != "" {
+ t.Fatalf("in-reply-to = %q", e.InReplyTo)
+ }
+ e = parse(t, "Subject: r\r\nReferences: \r\n\r\nx\r\n")
+ if e.InReplyTo != "" {
+ t.Fatalf("references fallback = %q", e.InReplyTo)
+ }
+ if e = parse(t, "Subject: r\r\n\r\nx\r\n"); e.InReplyTo != "" {
+ t.Fatalf("none expected, got %q", e.InReplyTo)
+ }
+}
+
+func TestBadOrMissingDateFallsBackToNow(t *testing.T) {
+ for _, h := range []string{"", "Date: not a date\r\n"} {
+ if e := parse(t, h+"Subject: d\r\n\r\nx\r\n"); !e.Date.Equal(now) {
+ t.Fatalf("%q: date = %v", h, e.Date)
+ }
+ }
+}
+
+func TestMissingSubject(t *testing.T) {
+ if e := parse(t, "From: a@x.com\r\n\r\nx\r\n"); e.Subject != "(no subject)" {
+ t.Fatalf("subject = %q", e.Subject)
+ }
+}
+
+func TestBodyIsCappedAndCleaned(t *testing.T) {
+ big := strings.Repeat("a", MaxBody+5000)
+ e := parse(t, "Subject: big\r\n\r\n"+big+"\r\n")
+ if len(e.Body) > MaxBody {
+ t.Fatalf("body len %d exceeds cap", len(e.Body))
+ }
+ e = parse(t, "Subject: nul\r\n\r\nab\x00cd\r\n")
+ if strings.Contains(e.Body, "\x00") {
+ t.Fatal("NUL must be removed")
+ }
+}
+
+func TestInvalidUTF8IsRepaired(t *testing.T) {
+ e := parse(t, "Subject: bad\r\nContent-Type: text/plain\r\n\r\nok \xff\xfe end\r\n")
+ if !strings.Contains(e.Body, "ok") || !strings.Contains(e.Body, "end") {
+ t.Fatalf("body = %q", e.Body)
+ }
+ if strings.ToValidUTF8(e.Body, "") != e.Body {
+ t.Fatal("body must be valid UTF-8")
+ }
+}
+
+func TestEmptyInputIsAnError(t *testing.T) {
+ if _, err := Parse(nil, now); err == nil {
+ t.Fatal("want error for an unparseable message")
+ }
+}
+
+func TestObfuscate(t *testing.T) {
+ cases := map[string]string{
+ "mail alice@example.com now": "mail alice@… now",
+ "a.b+c@sub.example.org, x@y.io": "a.b+c@…, x@…",
+ "no address here": "no address here",
+ "": "",
+ "twitter @handle only": "twitter @handle only",
+ }
+ for in, want := range cases {
+ if got := Obfuscate(in); got != want {
+ t.Errorf("Obfuscate(%q) = %q, want %q", in, got, want)
+ }
+ }
+}