parsimony/internal/mbox/mbox_test.go
tgiachi 9968c3b70f fix(mbox): stop buffering a line beyond the size limit
- readLine now holds at most max(limit+1, 1024) bytes of a line and reports its full
  length; the rest is read and thrown away, so a single 25 MiB line is no longer read whole.
- The message size is counted from the reported line lengths (after restoring >From),
  so the limit is unchanged for normal input.
- Tests: a line over the limit mid-file, at the end of the input and before the first
  separator, long lines under the limit, a tiny limit, exact boundary, readLine itself.
2026-11-10 11:53:32 +01:00

258 lines
9.1 KiB
Go

package mbox
import (
"errors"
"io"
"strings"
"testing"
"time"
)
func readAll(t *testing.T, input string, maxSize int) []Message {
t.Helper()
r := NewReader(strings.NewReader(input), maxSize)
var out []Message
for {
m, err := r.Next()
if err == io.EOF {
return out
}
if err != nil && !errors.Is(err, ErrTooLarge) {
t.Fatalf("next: %v", err)
}
m.Err = err
out = append(out, m)
}
}
const two = "From alice@example.com Tue Jan 2 15:04:05 2024\n" +
"From: alice@example.com\nSubject: one\n\nfirst body\n\n" +
"From bob@example.com Wed Jan 3 09:00:00 2024\n" +
"From: bob@example.com\nSubject: two\n\nsecond body\n"
func TestSplitsMessages(t *testing.T) {
ms := readAll(t, two, 0)
if len(ms) != 2 {
t.Fatalf("got %d messages", len(ms))
}
if string(ms[0].Data) != "From: alice@example.com\nSubject: one\n\nfirst body\n" {
t.Fatalf("first = %q", ms[0].Data)
}
if string(ms[1].Data) != "From: bob@example.com\nSubject: two\n\nsecond body\n" {
t.Fatalf("second = %q", ms[1].Data)
}
if ms[0].Envelope != "From alice@example.com Tue Jan 2 15:04:05 2024" {
t.Fatalf("envelope = %q", ms[0].Envelope)
}
}
func TestEnvelopeDate(t *testing.T) {
ms := readAll(t, two, 0)
want := time.Date(2024, 1, 2, 15, 4, 5, 0, time.UTC)
if got, ok := ms[0].Date(); !ok || !got.Equal(want) {
t.Fatalf("date = %v %v", got, ok)
}
for _, env := range []string{
"From x@y.z Tue Jan 2 15:04:05 +0200 2024",
"From x@y.z Tue Jan 2 15:04:05 2024",
"From x@y.z Tue Jan 2 15:04 2024",
"From MAILER-DAEMON Tue Jan 2 15:04:05 2024",
} {
if _, ok := (Message{Envelope: env}).Date(); !ok {
t.Errorf("%q must parse", env)
}
}
for _, env := range []string{"", "From x@y.z", "From x@y.z not a date"} {
if _, ok := (Message{Envelope: env}).Date(); ok {
t.Errorf("%q must not parse", env)
}
}
}
func TestProseLineStartingWithFromDoesNotSplit(t *testing.T) {
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nFrom now on we meet on Fridays.\nFrom: not a separator either\n"
ms := readAll(t, in, 0)
if len(ms) != 1 || !strings.Contains(string(ms[0].Data), "From now on we meet on Fridays.") {
t.Fatalf("messages: %d %q", len(ms), ms[0].Data)
}
}
func TestMboxrdUnescaping(t *testing.T) {
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n>From the log:\n>>From a quoted line\n> From not escaped\nplain\n"
ms := readAll(t, in, 0)
body := string(ms[0].Data)
if !strings.Contains(body, "\nFrom the log:\n") || !strings.Contains(body, "\n>From a quoted line\n") || !strings.Contains(body, "\n> From not escaped\n") {
t.Fatalf("body = %q", body)
}
}
func TestLeadingJunkAndEmptyInput(t *testing.T) {
if got := readAll(t, "", 0); len(got) != 0 {
t.Fatalf("empty input: %d", len(got))
}
if got := readAll(t, "\n\nsome preamble\nFrom a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nbody\n", 0); len(got) != 1 {
t.Fatalf("preamble must be ignored: %d", len(got))
}
if got := readAll(t, "just text, no separator at all\n", 0); len(got) != 0 {
t.Fatalf("no messages expected: %d", len(got))
}
}
func TestCRLFAndMissingFinalNewline(t *testing.T) {
in := "From a@x.com Tue Jan 2 15:04:05 2024\r\nSubject: s\r\n\r\nbody without newline"
ms := readAll(t, in, 0)
if len(ms) != 1 || !strings.HasSuffix(string(ms[0].Data), "body without newline") || !strings.Contains(string(ms[0].Data), "Subject: s\r\n") {
t.Fatalf("%q", ms[0].Data)
}
if string(readAll(t, "From a@x.com Tue Jan 2 15:04:05 2024\r\nSubject: one\r\n\r\nb\r\n\r\nFrom b@x.com Tue Jan 2 15:04:05 2024\r\nSubject: two\r\n\r\nb2\r\n", 0)[0].Data) != "Subject: one\r\n\r\nb\r\n" {
t.Fatal("the CRLF blank line before a separator belongs to the separator")
}
crlf := "From a@x.com Tue Jan 2 15:04:05 2024\r\nSubject: one\r\n\r\nb\r\n\r\nFrom b@x.com Tue Jan 2 15:04:05 2024\r\nSubject: two\r\n\r\nb2\r\n"
if got := readAll(t, crlf, 0); len(got) != 2 {
t.Fatalf("CRLF separators: %d", len(got))
}
}
func TestVeryLongLinesAreFine(t *testing.T) {
long := strings.Repeat("x", 300_000)
ms := readAll(t, "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n"+long+"\n", 0)
if len(ms) != 1 || !strings.Contains(string(ms[0].Data), long) {
t.Fatal("a long line must survive")
}
}
func TestOversizedMessagesAreReportedAndSkipped(t *testing.T) {
big := strings.Repeat("y\n", 5000)
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: ok1\n\nsmall\n" +
"From b@x.com Tue Jan 2 15:04:05 2024\nSubject: big\n\n" + big +
"From c@x.com Tue Jan 2 15:04:05 2024\nSubject: ok2\n\nsmall\n"
ms := readAll(t, in, 1000)
if len(ms) != 3 {
t.Fatalf("got %d", len(ms))
}
if ms[0].Err != nil || ms[2].Err != nil {
t.Fatalf("small messages must pass: %v %v", ms[0].Err, ms[2].Err)
}
if !errors.Is(ms[1].Err, ErrTooLarge) || ms[1].Data != nil {
t.Fatalf("big message: %v %d bytes", ms[1].Err, len(ms[1].Data))
}
if ms[1].Envelope != "From b@x.com Tue Jan 2 15:04:05 2024" || !strings.Contains(string(ms[2].Data), "ok2") {
t.Fatalf("the reader must resynchronise after a big message: %+v", ms[2])
}
}
func TestIndexIsOneBased(t *testing.T) {
ms := readAll(t, two, 0)
if ms[0].Index != 1 || ms[1].Index != 2 {
t.Fatalf("indexes: %d %d", ms[0].Index, ms[1].Index)
}
}
func TestReadErrorsArePropagated(t *testing.T) {
r := NewReader(io.MultiReader(strings.NewReader("From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nbody\n"), errReader{}), 0)
var last error
for i := 0; i < 5; i++ {
_, err := r.Next()
if err != nil {
last = err
break
}
}
if last == nil || last == io.EOF {
t.Fatalf("want the read error, got %v", last)
}
}
type errReader struct{}
func (errReader) Read([]byte) (int, error) { return 0, errors.New("disk on fire") }
func TestAVeryLongLineIsTooLargeAndTheReaderResynchronises(t *testing.T) {
long := strings.Repeat("a", 5000) // far above the limit of 100, with no way to be a separator
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n" + long + "\n\n" +
"From b@x.com Wed Jan 3 15:04:05 2024\nSubject: ok\n\nfine\n"
ms := readAll(t, in, 100)
if len(ms) != 2 || !errors.Is(ms[0].Err, ErrTooLarge) || ms[0].Data != nil {
t.Fatalf("messages: %+v", ms)
}
if ms[1].Err != nil || !strings.Contains(string(ms[1].Data), "fine") {
t.Fatalf("the next message must be read: %+v", ms[1])
}
}
func TestALongLineAtTheEndOfTheInputIsTooLarge(t *testing.T) {
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n" + strings.Repeat("a", 5000) // no newline
ms := readAll(t, in, 100)
if len(ms) != 1 || !errors.Is(ms[0].Err, ErrTooLarge) {
t.Fatalf("messages: %+v", ms)
}
}
func TestAnOverlongLineBeforeTheFirstSeparatorIsSkipped(t *testing.T) {
in := strings.Repeat("x", 5000) + "\nFrom a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nbody\n"
ms := readAll(t, in, 100)
if len(ms) != 1 || ms[0].Err != nil || !strings.Contains(string(ms[0].Data), "body") {
t.Fatalf("messages: %+v", ms)
}
}
func TestLongLinesBelowTheLimitStillWork(t *testing.T) {
long := strings.Repeat("a", 3000)
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n" + long + "\n"
ms := readAll(t, in, 4000)
if len(ms) != 1 || ms[0].Err != nil || !strings.Contains(string(ms[0].Data), long+"\n") {
t.Fatalf("messages: %+v", ms)
}
if ms = readAll(t, in, 0); len(ms) != 1 || ms[0].Err != nil {
t.Fatalf("no limit: %+v", ms)
}
}
func TestReadLineKeepsAtMostTheLimitButReportsTheFullLength(t *testing.T) {
// A line far longer than the reader's buffer: only keep bytes may be held.
in := strings.Repeat("a", 300_000) + "\nnext\n"
r := NewReader(strings.NewReader(in), 0)
line, n, err := r.readLine(1000)
if err != nil || n != 300_001 || len(line) > 1000 {
t.Fatalf("n=%d kept=%d err=%v", n, len(line), err)
}
if line, n, err = r.readLine(1000); err != nil || string(line) != "next\n" || n != 5 {
t.Fatalf("the following line: %q %d %v", line, n, err)
}
if _, _, err = r.readLine(1000); err != io.EOF {
t.Fatalf("want EOF, got %v", err)
}
// keep 0 means keep everything.
r = NewReader(strings.NewReader(in), 0)
if line, n, _ = r.readLine(0); len(line) != 300_001 || n != 300_001 {
t.Fatalf("unlimited: %d %d", len(line), n)
}
}
func TestASmallLimitDoesNotCutSeparatorsShort(t *testing.T) {
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nlong enough to be too large\n" +
"From b@x.com Wed Jan 3 15:04:05 2024\nok\n"
ms := readAll(t, in, 10)
if len(ms) != 2 || !errors.Is(ms[0].Err, ErrTooLarge) || ms[1].Err != nil || string(ms[1].Data) != "ok\n" {
t.Fatalf("messages: %+v", ms)
}
}
func TestKeepIsBoundedByTheLimit(t *testing.T) {
for _, c := range []struct{ max, want int }{{0, 0}, {-1, 0}, {100, minKeep}, {5000, 5001}} {
if got := NewReader(strings.NewReader(""), c.max).keep(); got != c.want {
t.Errorf("max %d: keep %d, want %d", c.max, got, c.want)
}
}
}
func TestTheLimitIsExactAndCountsTheRestoredLine(t *testing.T) {
body := "Subject: s\n\n>From here\n" // restored to "From here\n": 22 bytes
in := "From a@x.com Tue Jan 2 15:04:05 2024\n" + body
if ms := readAll(t, in, 22); len(ms) != 1 || ms[0].Err != nil {
t.Fatalf("exactly at the limit must pass: %+v", ms)
}
if ms := readAll(t, in, 21); len(ms) != 1 || !errors.Is(ms[0].Err, ErrTooLarge) {
t.Fatalf("one over the limit must be too large: %+v", ms)
}
}