fix(mbox): stop buffering a line beyond the size limit
- readLine now holds at most max(limit+1, 1024) bytes of a line and reports its full length; the rest is read and thrown away, so a single 25 MiB line is no longer read whole. - The message size is counted from the reported line lengths (after restoring >From), so the limit is unchanged for normal input. - Tests: a line over the limit mid-file, at the end of the input and before the first separator, long lines under the limit, a tiny limit, exact boundary, readLine itself.
This commit is contained in:
parent
79fe51b206
commit
9968c3b70f
2 changed files with 136 additions and 11 deletions
|
|
@ -67,14 +67,45 @@ func NewReader(r io.Reader, maxSize int) *Reader {
|
|||
return &Reader{br: bufio.NewReaderSize(r, 64*1024), max: maxSize}
|
||||
}
|
||||
|
||||
// readLine returns the next line including its newline. At the end of the input
|
||||
// it returns the last partial line (if any) with io.EOF on the following call.
|
||||
func (r *Reader) readLine() ([]byte, error) {
|
||||
line, err := r.br.ReadBytes('\n')
|
||||
if err == io.EOF && len(line) > 0 {
|
||||
return line, nil
|
||||
// minKeep is the least a line may be cut to, so a "From " separator is never cut short.
|
||||
const minKeep = 1024
|
||||
|
||||
// keep is how many bytes of a line are worth holding: a message with a longer line is too
|
||||
// large anyway, so the rest is read and thrown away instead of buffered.
|
||||
func (r *Reader) keep() int {
|
||||
if r.max <= 0 {
|
||||
return 0 // no limit
|
||||
}
|
||||
return max(r.max+1, minKeep)
|
||||
}
|
||||
|
||||
// readLine returns the next line including its newline, cut to keep bytes (0 means no cut),
|
||||
// and the full length of the line. At the end of the input it returns the last partial line
|
||||
// (if any) with io.EOF on the following call.
|
||||
func (r *Reader) readLine(keep int) ([]byte, int, error) {
|
||||
var line []byte
|
||||
n := 0
|
||||
for {
|
||||
chunk, err := r.br.ReadSlice('\n')
|
||||
n += len(chunk)
|
||||
if room := keep - len(line); keep == 0 || room > 0 {
|
||||
if keep > 0 && len(chunk) > room {
|
||||
chunk = chunk[:room]
|
||||
}
|
||||
line = append(line, chunk...)
|
||||
}
|
||||
switch err {
|
||||
case nil:
|
||||
return line, n, nil
|
||||
case bufio.ErrBufferFull:
|
||||
continue
|
||||
case io.EOF:
|
||||
if n > 0 {
|
||||
return line, n, nil
|
||||
}
|
||||
}
|
||||
return line, n, err
|
||||
}
|
||||
return line, err
|
||||
}
|
||||
|
||||
// Next returns the next message, or io.EOF when there are no more.
|
||||
|
|
@ -84,7 +115,7 @@ func (r *Reader) Next() (Message, error) {
|
|||
}
|
||||
// Skip anything before the first separator.
|
||||
for r.next == "" {
|
||||
line, err := r.readLine()
|
||||
line, _, err := r.readLine(r.keep())
|
||||
if err != nil {
|
||||
r.eof = true
|
||||
if err == io.EOF {
|
||||
|
|
@ -102,8 +133,9 @@ func (r *Reader) Next() (Message, error) {
|
|||
|
||||
var buf bytes.Buffer
|
||||
oversized := false
|
||||
size := 0
|
||||
for {
|
||||
line, err := r.readLine()
|
||||
line, n, err := r.readLine(r.keep())
|
||||
if err != nil {
|
||||
r.eof = true
|
||||
if err != io.EOF {
|
||||
|
|
@ -120,12 +152,15 @@ func (r *Reader) Next() (Message, error) {
|
|||
}
|
||||
if escaped.Match(line) {
|
||||
line = line[1:]
|
||||
n--
|
||||
}
|
||||
buf.Write(line)
|
||||
if r.max > 0 && buf.Len() > r.max {
|
||||
size += n
|
||||
if r.max > 0 && size > r.max {
|
||||
oversized = true
|
||||
buf.Reset()
|
||||
continue
|
||||
}
|
||||
buf.Write(line)
|
||||
}
|
||||
if oversized {
|
||||
msg.Err = ErrTooLarge
|
||||
|
|
|
|||
|
|
@ -166,3 +166,93 @@ func TestReadErrorsArePropagated(t *testing.T) {
|
|||
type errReader struct{}
|
||||
|
||||
func (errReader) Read([]byte) (int, error) { return 0, errors.New("disk on fire") }
|
||||
|
||||
func TestAVeryLongLineIsTooLargeAndTheReaderResynchronises(t *testing.T) {
|
||||
long := strings.Repeat("a", 5000) // far above the limit of 100, with no way to be a separator
|
||||
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n" + long + "\n\n" +
|
||||
"From b@x.com Wed Jan 3 15:04:05 2024\nSubject: ok\n\nfine\n"
|
||||
ms := readAll(t, in, 100)
|
||||
if len(ms) != 2 || !errors.Is(ms[0].Err, ErrTooLarge) || ms[0].Data != nil {
|
||||
t.Fatalf("messages: %+v", ms)
|
||||
}
|
||||
if ms[1].Err != nil || !strings.Contains(string(ms[1].Data), "fine") {
|
||||
t.Fatalf("the next message must be read: %+v", ms[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestALongLineAtTheEndOfTheInputIsTooLarge(t *testing.T) {
|
||||
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n" + strings.Repeat("a", 5000) // no newline
|
||||
ms := readAll(t, in, 100)
|
||||
if len(ms) != 1 || !errors.Is(ms[0].Err, ErrTooLarge) {
|
||||
t.Fatalf("messages: %+v", ms)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOverlongLineBeforeTheFirstSeparatorIsSkipped(t *testing.T) {
|
||||
in := strings.Repeat("x", 5000) + "\nFrom a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nbody\n"
|
||||
ms := readAll(t, in, 100)
|
||||
if len(ms) != 1 || ms[0].Err != nil || !strings.Contains(string(ms[0].Data), "body") {
|
||||
t.Fatalf("messages: %+v", ms)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLongLinesBelowTheLimitStillWork(t *testing.T) {
|
||||
long := strings.Repeat("a", 3000)
|
||||
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n" + long + "\n"
|
||||
ms := readAll(t, in, 4000)
|
||||
if len(ms) != 1 || ms[0].Err != nil || !strings.Contains(string(ms[0].Data), long+"\n") {
|
||||
t.Fatalf("messages: %+v", ms)
|
||||
}
|
||||
if ms = readAll(t, in, 0); len(ms) != 1 || ms[0].Err != nil {
|
||||
t.Fatalf("no limit: %+v", ms)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadLineKeepsAtMostTheLimitButReportsTheFullLength(t *testing.T) {
|
||||
// A line far longer than the reader's buffer: only keep bytes may be held.
|
||||
in := strings.Repeat("a", 300_000) + "\nnext\n"
|
||||
r := NewReader(strings.NewReader(in), 0)
|
||||
line, n, err := r.readLine(1000)
|
||||
if err != nil || n != 300_001 || len(line) > 1000 {
|
||||
t.Fatalf("n=%d kept=%d err=%v", n, len(line), err)
|
||||
}
|
||||
if line, n, err = r.readLine(1000); err != nil || string(line) != "next\n" || n != 5 {
|
||||
t.Fatalf("the following line: %q %d %v", line, n, err)
|
||||
}
|
||||
if _, _, err = r.readLine(1000); err != io.EOF {
|
||||
t.Fatalf("want EOF, got %v", err)
|
||||
}
|
||||
// keep 0 means keep everything.
|
||||
r = NewReader(strings.NewReader(in), 0)
|
||||
if line, n, _ = r.readLine(0); len(line) != 300_001 || n != 300_001 {
|
||||
t.Fatalf("unlimited: %d %d", len(line), n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASmallLimitDoesNotCutSeparatorsShort(t *testing.T) {
|
||||
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nlong enough to be too large\n" +
|
||||
"From b@x.com Wed Jan 3 15:04:05 2024\nok\n"
|
||||
ms := readAll(t, in, 10)
|
||||
if len(ms) != 2 || !errors.Is(ms[0].Err, ErrTooLarge) || ms[1].Err != nil || string(ms[1].Data) != "ok\n" {
|
||||
t.Fatalf("messages: %+v", ms)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeepIsBoundedByTheLimit(t *testing.T) {
|
||||
for _, c := range []struct{ max, want int }{{0, 0}, {-1, 0}, {100, minKeep}, {5000, 5001}} {
|
||||
if got := NewReader(strings.NewReader(""), c.max).keep(); got != c.want {
|
||||
t.Errorf("max %d: keep %d, want %d", c.max, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheLimitIsExactAndCountsTheRestoredLine(t *testing.T) {
|
||||
body := "Subject: s\n\n>From here\n" // restored to "From here\n": 22 bytes
|
||||
in := "From a@x.com Tue Jan 2 15:04:05 2024\n" + body
|
||||
if ms := readAll(t, in, 22); len(ms) != 1 || ms[0].Err != nil {
|
||||
t.Fatalf("exactly at the limit must pass: %+v", ms)
|
||||
}
|
||||
if ms := readAll(t, in, 21); len(ms) != 1 || !errors.Is(ms[0].Err, ErrTooLarge) {
|
||||
t.Fatalf("one over the limit must be too large: %+v", ms)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue