diff --git a/internal/mbox/mbox.go b/internal/mbox/mbox.go index a6aa4b6..1f86df9 100644 --- a/internal/mbox/mbox.go +++ b/internal/mbox/mbox.go @@ -67,14 +67,45 @@ func NewReader(r io.Reader, maxSize int) *Reader { return &Reader{br: bufio.NewReaderSize(r, 64*1024), max: maxSize} } -// readLine returns the next line including its newline. At the end of the input -// it returns the last partial line (if any) with io.EOF on the following call. -func (r *Reader) readLine() ([]byte, error) { - line, err := r.br.ReadBytes('\n') - if err == io.EOF && len(line) > 0 { - return line, nil +// minKeep is the least a line may be cut to, so a "From " separator is never cut short. +const minKeep = 1024 + +// keep is how many bytes of a line are worth holding: a message with a longer line is too +// large anyway, so the rest is read and thrown away instead of buffered. +func (r *Reader) keep() int { + if r.max <= 0 { + return 0 // no limit + } + return max(r.max+1, minKeep) +} + +// readLine returns the next line including its newline, cut to keep bytes (0 means no cut), +// and the full length of the line. At the end of the input it returns the last partial line +// (if any) with io.EOF on the following call. +func (r *Reader) readLine(keep int) ([]byte, int, error) { + var line []byte + n := 0 + for { + chunk, err := r.br.ReadSlice('\n') + n += len(chunk) + if room := keep - len(line); keep == 0 || room > 0 { + if keep > 0 && len(chunk) > room { + chunk = chunk[:room] + } + line = append(line, chunk...) + } + switch err { + case nil: + return line, n, nil + case bufio.ErrBufferFull: + continue + case io.EOF: + if n > 0 { + return line, n, nil + } + } + return line, n, err } - return line, err } // Next returns the next message, or io.EOF when there are no more. @@ -84,7 +115,7 @@ func (r *Reader) Next() (Message, error) { } // Skip anything before the first separator. for r.next == "" { - line, err := r.readLine() + line, _, err := r.readLine(r.keep()) if err != nil { r.eof = true if err == io.EOF { @@ -102,8 +133,9 @@ func (r *Reader) Next() (Message, error) { var buf bytes.Buffer oversized := false + size := 0 for { - line, err := r.readLine() + line, n, err := r.readLine(r.keep()) if err != nil { r.eof = true if err != io.EOF { @@ -120,12 +152,15 @@ func (r *Reader) Next() (Message, error) { } if escaped.Match(line) { line = line[1:] + n-- } - buf.Write(line) - if r.max > 0 && buf.Len() > r.max { + size += n + if r.max > 0 && size > r.max { oversized = true buf.Reset() + continue } + buf.Write(line) } if oversized { msg.Err = ErrTooLarge diff --git a/internal/mbox/mbox_test.go b/internal/mbox/mbox_test.go index a39e7e5..365d242 100644 --- a/internal/mbox/mbox_test.go +++ b/internal/mbox/mbox_test.go @@ -166,3 +166,93 @@ func TestReadErrorsArePropagated(t *testing.T) { type errReader struct{} func (errReader) Read([]byte) (int, error) { return 0, errors.New("disk on fire") } + +func TestAVeryLongLineIsTooLargeAndTheReaderResynchronises(t *testing.T) { + long := strings.Repeat("a", 5000) // far above the limit of 100, with no way to be a separator + in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n" + long + "\n\n" + + "From b@x.com Wed Jan 3 15:04:05 2024\nSubject: ok\n\nfine\n" + ms := readAll(t, in, 100) + if len(ms) != 2 || !errors.Is(ms[0].Err, ErrTooLarge) || ms[0].Data != nil { + t.Fatalf("messages: %+v", ms) + } + if ms[1].Err != nil || !strings.Contains(string(ms[1].Data), "fine") { + t.Fatalf("the next message must be read: %+v", ms[1]) + } +} + +func TestALongLineAtTheEndOfTheInputIsTooLarge(t *testing.T) { + in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n" + strings.Repeat("a", 5000) // no newline + ms := readAll(t, in, 100) + if len(ms) != 1 || !errors.Is(ms[0].Err, ErrTooLarge) { + t.Fatalf("messages: %+v", ms) + } +} + +func TestAnOverlongLineBeforeTheFirstSeparatorIsSkipped(t *testing.T) { + in := strings.Repeat("x", 5000) + "\nFrom a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nbody\n" + ms := readAll(t, in, 100) + if len(ms) != 1 || ms[0].Err != nil || !strings.Contains(string(ms[0].Data), "body") { + t.Fatalf("messages: %+v", ms) + } +} + +func TestLongLinesBelowTheLimitStillWork(t *testing.T) { + long := strings.Repeat("a", 3000) + in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n" + long + "\n" + ms := readAll(t, in, 4000) + if len(ms) != 1 || ms[0].Err != nil || !strings.Contains(string(ms[0].Data), long+"\n") { + t.Fatalf("messages: %+v", ms) + } + if ms = readAll(t, in, 0); len(ms) != 1 || ms[0].Err != nil { + t.Fatalf("no limit: %+v", ms) + } +} + +func TestReadLineKeepsAtMostTheLimitButReportsTheFullLength(t *testing.T) { + // A line far longer than the reader's buffer: only keep bytes may be held. + in := strings.Repeat("a", 300_000) + "\nnext\n" + r := NewReader(strings.NewReader(in), 0) + line, n, err := r.readLine(1000) + if err != nil || n != 300_001 || len(line) > 1000 { + t.Fatalf("n=%d kept=%d err=%v", n, len(line), err) + } + if line, n, err = r.readLine(1000); err != nil || string(line) != "next\n" || n != 5 { + t.Fatalf("the following line: %q %d %v", line, n, err) + } + if _, _, err = r.readLine(1000); err != io.EOF { + t.Fatalf("want EOF, got %v", err) + } + // keep 0 means keep everything. + r = NewReader(strings.NewReader(in), 0) + if line, n, _ = r.readLine(0); len(line) != 300_001 || n != 300_001 { + t.Fatalf("unlimited: %d %d", len(line), n) + } +} + +func TestASmallLimitDoesNotCutSeparatorsShort(t *testing.T) { + in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nlong enough to be too large\n" + + "From b@x.com Wed Jan 3 15:04:05 2024\nok\n" + ms := readAll(t, in, 10) + if len(ms) != 2 || !errors.Is(ms[0].Err, ErrTooLarge) || ms[1].Err != nil || string(ms[1].Data) != "ok\n" { + t.Fatalf("messages: %+v", ms) + } +} + +func TestKeepIsBoundedByTheLimit(t *testing.T) { + for _, c := range []struct{ max, want int }{{0, 0}, {-1, 0}, {100, minKeep}, {5000, 5001}} { + if got := NewReader(strings.NewReader(""), c.max).keep(); got != c.want { + t.Errorf("max %d: keep %d, want %d", c.max, got, c.want) + } + } +} + +func TestTheLimitIsExactAndCountsTheRestoredLine(t *testing.T) { + body := "Subject: s\n\n>From here\n" // restored to "From here\n": 22 bytes + in := "From a@x.com Tue Jan 2 15:04:05 2024\n" + body + if ms := readAll(t, in, 22); len(ms) != 1 || ms[0].Err != nil { + t.Fatalf("exactly at the limit must pass: %+v", ms) + } + if ms := readAll(t, in, 21); len(ms) != 1 || !errors.Is(ms[0].Err, ErrTooLarge) { + t.Fatalf("one over the limit must be too large: %+v", ms) + } +}