- it is linear (about 1.5 s per MiB under -race); a quadratic one would take minutes, so 15 s still catches it
411 lines
14 KiB
Go
411 lines
14 KiB
Go
package archive
|
|
|
|
import (
|
|
"fmt"
|
|
"runtime"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
var now = time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)
|
|
|
|
func parse(t *testing.T, raw string) Entry {
|
|
t.Helper()
|
|
e, err := Parse([]byte(raw), now)
|
|
if err != nil {
|
|
t.Fatalf("parse: %v", err)
|
|
}
|
|
return e
|
|
}
|
|
|
|
func TestPlainMessage(t *testing.T) {
|
|
e := parse(t, "From: Alice Smith <alice@example.com>\r\nSubject: Hello\r\nMessage-ID: <abc@x>\r\n"+
|
|
"Date: Mon, 02 Jan 2006 15:04:05 +0000\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nline one\r\nline two\r\n")
|
|
if e.Subject != "Hello" || e.Sender != "Alice Smith" || e.MessageID != "<abc@x>" {
|
|
t.Fatalf("entry: %+v", e)
|
|
}
|
|
if e.Body != "line one\nline two\n" {
|
|
t.Fatalf("body = %q", e.Body)
|
|
}
|
|
if !e.Date.Equal(time.Date(2006, 1, 2, 15, 4, 5, 0, time.UTC)) {
|
|
t.Fatalf("date = %v", e.Date)
|
|
}
|
|
}
|
|
|
|
func TestSenderWithoutNameNeverExposesTheDomain(t *testing.T) {
|
|
e := parse(t, "From: bob@secret.example.com\r\nSubject: x\r\n\r\nhi\r\n")
|
|
if e.Sender != "bob" {
|
|
t.Fatalf("sender = %q", e.Sender)
|
|
}
|
|
}
|
|
|
|
func TestEncodedSubjectAndSender(t *testing.T) {
|
|
e := parse(t, "From: =?UTF-8?B?w4lsw6g=?= <e@x.com>\r\nSubject: =?ISO-8859-1?Q?Caf=E9_ol=E9?=\r\n\r\nx\r\n")
|
|
if e.Subject != "Café olé" || e.Sender != "Él\u00e8" {
|
|
t.Fatalf("entry: %q %q", e.Subject, e.Sender)
|
|
}
|
|
}
|
|
|
|
func TestBase64UTF8Body(t *testing.T) {
|
|
e := parse(t, "Subject: b\r\nContent-Type: text/plain; charset=utf-8\r\nContent-Transfer-Encoding: base64\r\n\r\n"+
|
|
"Y2lhbyBtb25kbyDimIM=\r\n")
|
|
if strings.TrimSpace(e.Body) != "ciao mondo ☃" {
|
|
t.Fatalf("body = %q", e.Body)
|
|
}
|
|
}
|
|
|
|
func TestQuotedPrintableLatin1Body(t *testing.T) {
|
|
e := parse(t, "Subject: q\r\nContent-Type: text/plain; charset=iso-8859-1\r\nContent-Transfer-Encoding: quoted-printable\r\n\r\n"+
|
|
"caff=E8 l=E0\r\n")
|
|
if strings.TrimSpace(e.Body) != "caffè là" {
|
|
t.Fatalf("body = %q", e.Body)
|
|
}
|
|
}
|
|
|
|
func TestMultipartAlternativePrefersPlain(t *testing.T) {
|
|
e := parse(t, "Subject: m\r\nContent-Type: multipart/alternative; boundary=BB\r\n\r\n"+
|
|
"--BB\r\nContent-Type: text/html\r\n\r\n<p>html version</p>\r\n"+
|
|
"--BB\r\nContent-Type: text/plain\r\n\r\nplain version\r\n--BB--\r\n")
|
|
if strings.TrimSpace(e.Body) != "plain version" {
|
|
t.Fatalf("body = %q", e.Body)
|
|
}
|
|
}
|
|
|
|
func TestMultipartMixedWithAttachmentAndNesting(t *testing.T) {
|
|
e := parse(t, "Subject: m\r\nContent-Type: multipart/mixed; boundary=OUT\r\n\r\n"+
|
|
"--OUT\r\nContent-Type: multipart/alternative; boundary=IN\r\n\r\n"+
|
|
"--IN\r\nContent-Type: text/plain\r\n\r\nreal text\r\n--IN--\r\n"+
|
|
"--OUT\r\nContent-Type: application/pdf\r\nContent-Disposition: attachment; filename=a.pdf\r\nContent-Transfer-Encoding: base64\r\n\r\nJVBERi0=\r\n--OUT--\r\n")
|
|
if strings.TrimSpace(e.Body) != "real text" {
|
|
t.Fatalf("body = %q", e.Body)
|
|
}
|
|
if strings.Contains(e.Body, "JVBER") {
|
|
t.Fatal("attachments must not leak into the body")
|
|
}
|
|
}
|
|
|
|
func TestHTMLOnlyIsStripped(t *testing.T) {
|
|
e := parse(t, "Subject: h\r\nContent-Type: text/html\r\n\r\n"+
|
|
"<html><head><style>p{color:red}</style></head><body><script>alert(1)</script><p>Hello & <b>welcome</b></p><p>second</p></body></html>\r\n")
|
|
if strings.ContainsAny(e.Body, "<>") || strings.Contains(e.Body, "alert") || strings.Contains(e.Body, "color") {
|
|
t.Fatalf("html not stripped: %q", e.Body)
|
|
}
|
|
if !strings.Contains(e.Body, "Hello & welcome") || !strings.Contains(e.Body, "second") {
|
|
t.Fatalf("text lost: %q", e.Body)
|
|
}
|
|
}
|
|
|
|
func TestUnknownCharsetStillArchives(t *testing.T) {
|
|
e := parse(t, "Subject: u\r\nContent-Type: text/plain; charset=x-made-up\r\n\r\nstill here\r\n")
|
|
if !strings.Contains(e.Body, "still here") {
|
|
t.Fatalf("body = %q", e.Body)
|
|
}
|
|
}
|
|
|
|
func TestMissingMessageIDIsSynthesizedAndStable(t *testing.T) {
|
|
raw := "From: a@x.com\r\nSubject: no id\r\n\r\nbody\r\n"
|
|
a, b := parse(t, raw), parse(t, raw)
|
|
if a.MessageID == "" || !strings.HasPrefix(a.MessageID, "<") || a.MessageID != b.MessageID {
|
|
t.Fatalf("ids: %q %q", a.MessageID, b.MessageID)
|
|
}
|
|
if c := parse(t, raw+"x"); c.MessageID == a.MessageID {
|
|
t.Fatal("different messages need different ids")
|
|
}
|
|
}
|
|
|
|
func TestInReplyToAndReferences(t *testing.T) {
|
|
e := parse(t, "Subject: r\r\nIn-Reply-To: <parent@x> (comment)\r\n\r\nx\r\n")
|
|
if e.InReplyTo != "<parent@x>" {
|
|
t.Fatalf("in-reply-to = %q", e.InReplyTo)
|
|
}
|
|
e = parse(t, "Subject: r\r\nReferences: <root@x> <mid@x> <last@x>\r\n\r\nx\r\n")
|
|
if e.InReplyTo != "<last@x>" {
|
|
t.Fatalf("references fallback = %q", e.InReplyTo)
|
|
}
|
|
if e = parse(t, "Subject: r\r\n\r\nx\r\n"); e.InReplyTo != "" {
|
|
t.Fatalf("none expected, got %q", e.InReplyTo)
|
|
}
|
|
}
|
|
|
|
func TestBadOrMissingDateFallsBackToNow(t *testing.T) {
|
|
for _, h := range []string{"", "Date: not a date\r\n"} {
|
|
if e := parse(t, h+"Subject: d\r\n\r\nx\r\n"); !e.Date.Equal(now) {
|
|
t.Fatalf("%q: date = %v", h, e.Date)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestMissingSubject(t *testing.T) {
|
|
if e := parse(t, "From: a@x.com\r\n\r\nx\r\n"); e.Subject != "(no subject)" {
|
|
t.Fatalf("subject = %q", e.Subject)
|
|
}
|
|
}
|
|
|
|
func TestBodyIsCappedAndCleaned(t *testing.T) {
|
|
big := strings.Repeat("a", MaxBody+5000)
|
|
e := parse(t, "Subject: big\r\n\r\n"+big+"\r\n")
|
|
if len(e.Body) > MaxBody {
|
|
t.Fatalf("body len %d exceeds cap", len(e.Body))
|
|
}
|
|
e = parse(t, "Subject: nul\r\n\r\nab\x00cd\r\n")
|
|
if strings.Contains(e.Body, "\x00") {
|
|
t.Fatal("NUL must be removed")
|
|
}
|
|
}
|
|
|
|
func TestInvalidUTF8IsRepaired(t *testing.T) {
|
|
e := parse(t, "Subject: bad\r\nContent-Type: text/plain\r\n\r\nok \xff\xfe end\r\n")
|
|
if !strings.Contains(e.Body, "ok") || !strings.Contains(e.Body, "end") {
|
|
t.Fatalf("body = %q", e.Body)
|
|
}
|
|
if strings.ToValidUTF8(e.Body, "") != e.Body {
|
|
t.Fatal("body must be valid UTF-8")
|
|
}
|
|
}
|
|
|
|
func TestEmptyInputIsAnError(t *testing.T) {
|
|
if _, err := Parse(nil, now); err == nil {
|
|
t.Fatal("want error for an unparseable message")
|
|
}
|
|
}
|
|
|
|
func TestObfuscate(t *testing.T) {
|
|
cases := map[string]string{
|
|
"mail alice@example.com now": "mail alice@… now",
|
|
"a.b+c@sub.example.org, x@y.io": "a.b+c@…, x@…",
|
|
"no address here": "no address here",
|
|
"<bob@x.co>": "<bob@…>",
|
|
"twitter @handle only": "twitter @handle only",
|
|
}
|
|
for in, want := range cases {
|
|
if got := Obfuscate(in); got != want {
|
|
t.Errorf("Obfuscate(%q) = %q, want %q", in, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestTextAttachmentIsNotTheBody(t *testing.T) {
|
|
e := parse(t, "Subject: m\r\nContent-Type: multipart/mixed; boundary=B\r\n\r\n"+
|
|
"--B\r\nContent-Type: text/plain\r\nContent-Disposition: attachment; filename=log.txt\r\n\r\nattached log\r\n"+
|
|
"--B\r\nContent-Type: text/plain\r\n\r\nthe real message\r\n--B--\r\n")
|
|
if strings.TrimSpace(e.Body) != "the real message" {
|
|
t.Fatalf("body = %q", e.Body)
|
|
}
|
|
}
|
|
|
|
func TestFirstTextPartWins(t *testing.T) {
|
|
e := parse(t, "Subject: m\r\nContent-Type: multipart/mixed; boundary=B\r\n\r\n"+
|
|
"--B\r\nContent-Type: text/plain\r\n\r\nmessage\r\n"+
|
|
"--B\r\nContent-Type: text/plain\r\n\r\n-- \r\nsignature footer\r\n--B--\r\n")
|
|
if strings.TrimSpace(e.Body) != "message" {
|
|
t.Fatalf("body = %q", e.Body)
|
|
}
|
|
}
|
|
|
|
func TestCapNeverSplitsAMultibyteCharacter(t *testing.T) {
|
|
// 3-byte characters: MaxBody is not a multiple of 3, so a blind cut lands mid-character.
|
|
e := parse(t, "Subject: big\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n"+strings.Repeat("€", MaxBody/3+10)+"\r\n")
|
|
if !utf8.ValidString(e.Body) || strings.ContainsRune(e.Body, utf8.RuneError) {
|
|
t.Fatal("body was cut in the middle of a character")
|
|
}
|
|
if len(e.Body) > MaxBody || len(e.Body) < MaxBody-3 {
|
|
t.Fatalf("body len %d", len(e.Body))
|
|
}
|
|
}
|
|
|
|
// nestedBomb is a message with a text part followed by a part that nests
|
|
// multipart/mixed levels deep.
|
|
func nestedBomb(levels int) string {
|
|
var b strings.Builder
|
|
b.WriteString("From: a@example.com\r\nSubject: bomb\r\nContent-Type: multipart/mixed; boundary=b0\r\n\r\n")
|
|
b.WriteString("--b0\r\nContent-Type: text/plain\r\n\r\nhello\r\n")
|
|
for i := 0; i < levels; i++ {
|
|
fmt.Fprintf(&b, "--b%d\r\nContent-Type: multipart/mixed; boundary=b%d\r\n\r\n", i, i+1)
|
|
}
|
|
b.WriteString("--b" + fmt.Sprint(levels) + "\r\nContent-Type: text/plain\r\n\r\ndeep\r\n")
|
|
return b.String()
|
|
}
|
|
|
|
func parseWithin(t *testing.T, raw string, limit time.Duration) Entry {
|
|
t.Helper()
|
|
type result struct {
|
|
e Entry
|
|
err error
|
|
}
|
|
done := make(chan result, 1)
|
|
go func() {
|
|
e, err := Parse([]byte(raw), now)
|
|
done <- result{e, err}
|
|
}()
|
|
select {
|
|
case r := <-done:
|
|
if r.err != nil {
|
|
t.Fatalf("parse: %v", r.err)
|
|
}
|
|
return r.e
|
|
case <-time.After(limit):
|
|
t.Fatalf("Parse still running after %v", limit)
|
|
return Entry{}
|
|
}
|
|
}
|
|
|
|
func TestDeeplyNestedMultipartIsBounded(t *testing.T) {
|
|
e := parseWithin(t, nestedBomb(20000), 2*time.Second)
|
|
if e.Body != "hello\n" && e.Body != "hello" {
|
|
t.Fatalf("body = %q", e.Body)
|
|
}
|
|
}
|
|
|
|
func TestNestingDeeperThanTheLimitIsIgnored(t *testing.T) {
|
|
// text/plain at depth 3 is found, at depth 40 it is not.
|
|
shallow := "Content-Type: multipart/mixed; boundary=b0\r\n\r\n" +
|
|
"--b0\r\nContent-Type: multipart/mixed; boundary=b1\r\n\r\n" +
|
|
"--b1\r\nContent-Type: multipart/mixed; boundary=b2\r\n\r\n" +
|
|
"--b2\r\nContent-Type: text/plain\r\n\r\nfound\r\n"
|
|
if e := parse(t, "Subject: s\r\n"+shallow); !strings.Contains(e.Body, "found") {
|
|
t.Fatalf("shallow body = %q", e.Body)
|
|
}
|
|
if e := parse(t, strings.Replace(nestedBomb(40), "text/plain\r\n\r\nhello", "application/octet-stream\r\n\r\nhello", 1)); strings.Contains(e.Body, "deep") {
|
|
t.Fatalf("a part 40 levels deep was read: %q", e.Body)
|
|
}
|
|
}
|
|
|
|
func TestManyPartsAreBounded(t *testing.T) {
|
|
var b strings.Builder
|
|
b.WriteString("Subject: many\r\nContent-Type: multipart/mixed; boundary=b\r\n\r\n")
|
|
for i := 0; i < 5000; i++ {
|
|
b.WriteString("--b\r\nContent-Type: application/octet-stream\r\n\r\nx\r\n")
|
|
}
|
|
b.WriteString("--b\r\nContent-Type: text/plain\r\n\r\nlast\r\n")
|
|
e := parseWithin(t, b.String(), 2*time.Second)
|
|
if strings.Contains(e.Body, "last") {
|
|
t.Fatalf("part 5001 was read: %q", e.Body)
|
|
}
|
|
|
|
// A text part among the first parts is still found.
|
|
b.Reset()
|
|
b.WriteString("Subject: many\r\nContent-Type: multipart/mixed; boundary=b\r\n\r\n")
|
|
for i := 0; i < 50; i++ {
|
|
b.WriteString("--b\r\nContent-Type: application/octet-stream\r\n\r\nx\r\n")
|
|
}
|
|
b.WriteString("--b\r\nContent-Type: text/plain\r\n\r\nearly\r\n")
|
|
if e := parse(t, b.String()); !strings.Contains(e.Body, "early") {
|
|
t.Fatalf("body = %q", e.Body)
|
|
}
|
|
}
|
|
|
|
func TestOnlyTheTextPartsThatAreNeededAreRead(t *testing.T) {
|
|
// 30 text parts of 1 MiB each: the first wins and the rest are not read.
|
|
big := strings.Repeat("a", MaxBody)
|
|
var b strings.Builder
|
|
b.WriteString("Subject: big\r\nContent-Type: multipart/mixed; boundary=b\r\n\r\n")
|
|
for i := 0; i < 30; i++ {
|
|
b.WriteString("--b\r\nContent-Type: text/plain\r\n\r\n" + big + "\r\n")
|
|
}
|
|
raw := []byte(b.String())
|
|
var before, after runtime.MemStats
|
|
runtime.ReadMemStats(&before)
|
|
e, err := Parse(raw, now)
|
|
runtime.ReadMemStats(&after)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(e.Body) != MaxBody {
|
|
t.Fatalf("body = %d bytes", len(e.Body))
|
|
}
|
|
if used := after.TotalAlloc - before.TotalAlloc; used > 24<<20 {
|
|
t.Fatalf("Parse allocated %d MiB for a 1 MiB body", used>>20)
|
|
}
|
|
}
|
|
|
|
func TestPathologicalHTMLIsNotQuadratic(t *testing.T) {
|
|
for name, frag := range map[string]string{
|
|
"open script": "<script>",
|
|
"open tag": "<a ",
|
|
"open style": "<style x>",
|
|
"break": "<br",
|
|
} {
|
|
html := strings.Repeat(frag, MaxBody/len(frag))
|
|
raw := "Subject: h\r\nContent-Type: text/html\r\n\r\n" + html
|
|
t.Run(name, func(t *testing.T) { parseWithin(t, raw, 15*time.Second) })
|
|
}
|
|
}
|
|
|
|
func TestFarFutureDateIsClamped(t *testing.T) {
|
|
// More than 24 hours ahead: the time of parsing. A little ahead is kept.
|
|
e := parse(t, "Date: Mon, 02 Jan 2090 15:04:05 +0000\r\nSubject: d\r\n\r\nx\r\n")
|
|
if !e.Date.Equal(now) {
|
|
t.Fatalf("far future date = %v", e.Date)
|
|
}
|
|
soon := now.Add(23 * time.Hour).Format(time.RFC1123Z)
|
|
if e := parse(t, "Date: "+soon+"\r\nSubject: d\r\n\r\nx\r\n"); e.Date.Equal(now) {
|
|
t.Fatalf("a date 23 hours ahead was clamped: %v", e.Date)
|
|
}
|
|
late := now.Add(25 * time.Hour).Format(time.RFC1123Z)
|
|
if e := parse(t, "Date: "+late+"\r\nSubject: d\r\n\r\nx\r\n"); !e.Date.Equal(now) {
|
|
t.Fatalf("a date 25 hours ahead = %v", e.Date)
|
|
}
|
|
}
|
|
|
|
func TestDateBefore1990IsMissing(t *testing.T) {
|
|
for _, d := range []string{"Sun, 31 Dec 1989 23:59:59 +0000", "Thu, 01 Jan 1970 00:00:00 +0000", "Mon, 01 Jan 1900 00:00:00 +0000"} {
|
|
if e := parse(t, "Date: "+d+"\r\nSubject: d\r\n\r\nx\r\n"); !e.Date.Equal(now) {
|
|
t.Fatalf("%s: date = %v", d, e.Date)
|
|
}
|
|
}
|
|
e := parse(t, "Date: Mon, 01 Jan 1990 00:00:00 +0000\r\nSubject: d\r\n\r\nx\r\n")
|
|
if !e.Date.Equal(time.Date(1990, 1, 1, 0, 0, 0, 0, time.UTC)) {
|
|
t.Fatalf("1990 date = %v", e.Date)
|
|
}
|
|
}
|
|
|
|
// textAtDepth is a message whose only text part sits inside n nested
|
|
// multipart levels.
|
|
func textAtDepth(n int) string {
|
|
var b strings.Builder
|
|
b.WriteString("Subject: s\r\nContent-Type: multipart/mixed; boundary=b0\r\n\r\n")
|
|
for i := 0; i < n-1; i++ {
|
|
fmt.Fprintf(&b, "--b%d\r\nContent-Type: multipart/mixed; boundary=b%d\r\n\r\n", i, i+1)
|
|
}
|
|
fmt.Fprintf(&b, "--b%d\r\nContent-Type: text/plain\r\n\r\nfound\r\n", n-1)
|
|
return b.String()
|
|
}
|
|
|
|
func TestNestingLimitIsTenLevels(t *testing.T) {
|
|
if e := parse(t, textAtDepth(10)); !strings.Contains(e.Body, "found") {
|
|
t.Fatalf("10 levels: body = %q", e.Body)
|
|
}
|
|
if e := parse(t, textAtDepth(11)); e.Body != "" {
|
|
t.Fatalf("11 levels: body = %q", e.Body)
|
|
}
|
|
}
|
|
|
|
func TestPartLimitIsOneHundred(t *testing.T) {
|
|
withTextAt := func(n int) string {
|
|
var b strings.Builder
|
|
b.WriteString("Subject: s\r\nContent-Type: multipart/mixed; boundary=b\r\n\r\n")
|
|
for i := 1; i < n; i++ {
|
|
b.WriteString("--b\r\nContent-Type: application/octet-stream\r\n\r\nx\r\n")
|
|
}
|
|
b.WriteString("--b\r\nContent-Type: text/plain\r\n\r\nfound\r\n")
|
|
return b.String()
|
|
}
|
|
if e := parse(t, withTextAt(100)); !strings.Contains(e.Body, "found") {
|
|
t.Fatalf("part 100: body = %q", e.Body)
|
|
}
|
|
if e := parse(t, withTextAt(101)); e.Body != "" {
|
|
t.Fatalf("part 101: body = %q", e.Body)
|
|
}
|
|
}
|
|
|
|
func TestFirstPartOfEachKindWinsEvenWhenPlainIsBlank(t *testing.T) {
|
|
part := func(ct, body string) string { return "--b\r\nContent-Type: " + ct + "\r\n\r\n" + body + "\r\n" }
|
|
raw := "Subject: s\r\nContent-Type: multipart/mixed; boundary=b\r\n\r\n" +
|
|
part("text/plain", " ") + part("text/plain", "second plain") +
|
|
part("text/html", "<p>first html</p>") + part("text/html", "<p>second html</p>")
|
|
if e := parse(t, raw); strings.TrimSpace(e.Body) != "first html" {
|
|
t.Fatalf("body = %q", e.Body)
|
|
}
|
|
}
|