fix(mbox): stop buffering a line beyond the size limit

- readLine now holds at most max(limit+1, 1024) bytes of a line and reports its full
  length; the rest is read and thrown away, so a single 25 MiB line is no longer read whole.
- The message size is counted from the reported line lengths (after restoring >From),
  so the limit is unchanged for normal input.
- Tests: a line over the limit mid-file, at the end of the input and before the first
  separator, long lines under the limit, a tiny limit, exact boundary, readLine itself.
This commit is contained in:
tgiachi 2026-11-10 11:53:32 +01:00
commit 9968c3b70f
2 changed files with 136 additions and 11 deletions

View file

@ -67,14 +67,45 @@ func NewReader(r io.Reader, maxSize int) *Reader {
return &Reader{br: bufio.NewReaderSize(r, 64*1024), max: maxSize}
}
// readLine returns the next line including its newline. At the end of the input
// it returns the last partial line (if any) with io.EOF on the following call.
func (r *Reader) readLine() ([]byte, error) {
line, err := r.br.ReadBytes('\n')
if err == io.EOF && len(line) > 0 {
return line, nil
// minKeep is the least a line may be cut to, so a "From " separator is never cut short.
const minKeep = 1024
// keep is how many bytes of a line are worth holding: a message with a longer line is too
// large anyway, so the rest is read and thrown away instead of buffered.
func (r *Reader) keep() int {
if r.max <= 0 {
return 0 // no limit
}
return max(r.max+1, minKeep)
}
// readLine returns the next line including its newline, cut to keep bytes (0 means no cut),
// and the full length of the line. At the end of the input it returns the last partial line
// (if any) with io.EOF on the following call.
func (r *Reader) readLine(keep int) ([]byte, int, error) {
var line []byte
n := 0
for {
chunk, err := r.br.ReadSlice('\n')
n += len(chunk)
if room := keep - len(line); keep == 0 || room > 0 {
if keep > 0 && len(chunk) > room {
chunk = chunk[:room]
}
line = append(line, chunk...)
}
switch err {
case nil:
return line, n, nil
case bufio.ErrBufferFull:
continue
case io.EOF:
if n > 0 {
return line, n, nil
}
}
return line, n, err
}
return line, err
}
// Next returns the next message, or io.EOF when there are no more.
@ -84,7 +115,7 @@ func (r *Reader) Next() (Message, error) {
}
// Skip anything before the first separator.
for r.next == "" {
line, err := r.readLine()
line, _, err := r.readLine(r.keep())
if err != nil {
r.eof = true
if err == io.EOF {
@ -102,8 +133,9 @@ func (r *Reader) Next() (Message, error) {
var buf bytes.Buffer
oversized := false
size := 0
for {
line, err := r.readLine()
line, n, err := r.readLine(r.keep())
if err != nil {
r.eof = true
if err != io.EOF {
@ -120,12 +152,15 @@ func (r *Reader) Next() (Message, error) {
}
if escaped.Match(line) {
line = line[1:]
n--
}
buf.Write(line)
if r.max > 0 && buf.Len() > r.max {
size += n
if r.max > 0 && size > r.max {
oversized = true
buf.Reset()
continue
}
buf.Write(line)
}
if oversized {
msg.Err = ErrTooLarge

View file

@ -166,3 +166,93 @@ func TestReadErrorsArePropagated(t *testing.T) {
type errReader struct{}
func (errReader) Read([]byte) (int, error) { return 0, errors.New("disk on fire") }
func TestAVeryLongLineIsTooLargeAndTheReaderResynchronises(t *testing.T) {
long := strings.Repeat("a", 5000) // far above the limit of 100, with no way to be a separator
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n" + long + "\n\n" +
"From b@x.com Wed Jan 3 15:04:05 2024\nSubject: ok\n\nfine\n"
ms := readAll(t, in, 100)
if len(ms) != 2 || !errors.Is(ms[0].Err, ErrTooLarge) || ms[0].Data != nil {
t.Fatalf("messages: %+v", ms)
}
if ms[1].Err != nil || !strings.Contains(string(ms[1].Data), "fine") {
t.Fatalf("the next message must be read: %+v", ms[1])
}
}
func TestALongLineAtTheEndOfTheInputIsTooLarge(t *testing.T) {
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n" + strings.Repeat("a", 5000) // no newline
ms := readAll(t, in, 100)
if len(ms) != 1 || !errors.Is(ms[0].Err, ErrTooLarge) {
t.Fatalf("messages: %+v", ms)
}
}
func TestAnOverlongLineBeforeTheFirstSeparatorIsSkipped(t *testing.T) {
in := strings.Repeat("x", 5000) + "\nFrom a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nbody\n"
ms := readAll(t, in, 100)
if len(ms) != 1 || ms[0].Err != nil || !strings.Contains(string(ms[0].Data), "body") {
t.Fatalf("messages: %+v", ms)
}
}
func TestLongLinesBelowTheLimitStillWork(t *testing.T) {
long := strings.Repeat("a", 3000)
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n" + long + "\n"
ms := readAll(t, in, 4000)
if len(ms) != 1 || ms[0].Err != nil || !strings.Contains(string(ms[0].Data), long+"\n") {
t.Fatalf("messages: %+v", ms)
}
if ms = readAll(t, in, 0); len(ms) != 1 || ms[0].Err != nil {
t.Fatalf("no limit: %+v", ms)
}
}
func TestReadLineKeepsAtMostTheLimitButReportsTheFullLength(t *testing.T) {
// A line far longer than the reader's buffer: only keep bytes may be held.
in := strings.Repeat("a", 300_000) + "\nnext\n"
r := NewReader(strings.NewReader(in), 0)
line, n, err := r.readLine(1000)
if err != nil || n != 300_001 || len(line) > 1000 {
t.Fatalf("n=%d kept=%d err=%v", n, len(line), err)
}
if line, n, err = r.readLine(1000); err != nil || string(line) != "next\n" || n != 5 {
t.Fatalf("the following line: %q %d %v", line, n, err)
}
if _, _, err = r.readLine(1000); err != io.EOF {
t.Fatalf("want EOF, got %v", err)
}
// keep 0 means keep everything.
r = NewReader(strings.NewReader(in), 0)
if line, n, _ = r.readLine(0); len(line) != 300_001 || n != 300_001 {
t.Fatalf("unlimited: %d %d", len(line), n)
}
}
func TestASmallLimitDoesNotCutSeparatorsShort(t *testing.T) {
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nlong enough to be too large\n" +
"From b@x.com Wed Jan 3 15:04:05 2024\nok\n"
ms := readAll(t, in, 10)
if len(ms) != 2 || !errors.Is(ms[0].Err, ErrTooLarge) || ms[1].Err != nil || string(ms[1].Data) != "ok\n" {
t.Fatalf("messages: %+v", ms)
}
}
func TestKeepIsBoundedByTheLimit(t *testing.T) {
for _, c := range []struct{ max, want int }{{0, 0}, {-1, 0}, {100, minKeep}, {5000, 5001}} {
if got := NewReader(strings.NewReader(""), c.max).keep(); got != c.want {
t.Errorf("max %d: keep %d, want %d", c.max, got, c.want)
}
}
}
func TestTheLimitIsExactAndCountsTheRestoredLine(t *testing.T) {
body := "Subject: s\n\n>From here\n" // restored to "From here\n": 22 bytes
in := "From a@x.com Tue Jan 2 15:04:05 2024\n" + body
if ms := readAll(t, in, 22); len(ms) != 1 || ms[0].Err != nil {
t.Fatalf("exactly at the limit must pass: %+v", ms)
}
if ms := readAll(t, in, 21); len(ms) != 1 || !errors.Is(ms[0].Err, ErrTooLarge) {
t.Fatalf("one over the limit must be too large: %+v", ms)
}
}