parsimony/internal/mbox/mbox.go
tgiachi 9968c3b70f fix(mbox): stop buffering a line beyond the size limit
- readLine now holds at most max(limit+1, 1024) bytes of a line and reports its full
  length; the rest is read and thrown away, so a single 25 MiB line is no longer read whole.
- The message size is counted from the reported line lengths (after restoring >From),
  so the limit is unchanged for normal input.
- Tests: a line over the limit mid-file, at the end of the input and before the first
  separator, long lines under the limit, a tiny limit, exact boundary, readLine itself.
2026-11-10 11:53:32 +01:00

182 lines
4.5 KiB
Go

// Package mbox reads mbox archives, such as the ones Mailman exports.
//
// A message starts at a line like "From alice@example.com Tue Jan 2 15:04:05 2024".
// A line that merely begins with "From " in prose does not split a message. Lines
// escaped as ">From " are restored (mboxrd); the blank line before a separator
// belongs to the separator, not to the message.
package mbox
import (
"bufio"
"bytes"
"errors"
"io"
"regexp"
"strings"
"time"
)
// ErrTooLarge is returned with a message that exceeded the size limit and was skipped.
var ErrTooLarge = errors.New("message too large")
var (
separator = regexp.MustCompile(`^From \S+\s+(Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2}\s+\d{1,2}:\d{2}`)
escaped = regexp.MustCompile(`^>+From `)
)
// Message is one message of the mbox.
type Message struct {
Index int // 1-based position in the file
Envelope string // the "From " line
Data []byte // the RFC 5322 message; nil when Err is set
Err error
}
// Date returns the date of the "From " line, when it has a parsable one.
func (m Message) Date() (time.Time, bool) {
f := strings.Fields(m.Envelope)
if len(f) < 3 || f[0] != "From" {
return time.Time{}, false
}
rest := strings.Join(f[2:], " ")
for _, layout := range []string{
"Mon Jan 2 15:04:05 2006",
"Mon Jan 2 15:04:05 -0700 2006",
"Mon Jan 2 15:04:05 MST 2006",
"Mon Jan 2 15:04 2006",
} {
if t, err := time.Parse(layout, rest); err == nil {
return t, true
}
}
return time.Time{}, false
}
// Reader returns the messages of an mbox one at a time.
type Reader struct {
br *bufio.Reader
max int
next string // envelope of the message that comes next, once seen
index int
eof bool
}
// NewReader reads from r. Messages larger than maxSize bytes (0 means no limit)
// are skipped and reported with ErrTooLarge.
func NewReader(r io.Reader, maxSize int) *Reader {
return &Reader{br: bufio.NewReaderSize(r, 64*1024), max: maxSize}
}
// minKeep is the least a line may be cut to, so a "From " separator is never cut short.
const minKeep = 1024
// keep is how many bytes of a line are worth holding: a message with a longer line is too
// large anyway, so the rest is read and thrown away instead of buffered.
func (r *Reader) keep() int {
if r.max <= 0 {
return 0 // no limit
}
return max(r.max+1, minKeep)
}
// readLine returns the next line including its newline, cut to keep bytes (0 means no cut),
// and the full length of the line. At the end of the input it returns the last partial line
// (if any) with io.EOF on the following call.
func (r *Reader) readLine(keep int) ([]byte, int, error) {
var line []byte
n := 0
for {
chunk, err := r.br.ReadSlice('\n')
n += len(chunk)
if room := keep - len(line); keep == 0 || room > 0 {
if keep > 0 && len(chunk) > room {
chunk = chunk[:room]
}
line = append(line, chunk...)
}
switch err {
case nil:
return line, n, nil
case bufio.ErrBufferFull:
continue
case io.EOF:
if n > 0 {
return line, n, nil
}
}
return line, n, err
}
}
// Next returns the next message, or io.EOF when there are no more.
func (r *Reader) Next() (Message, error) {
if r.eof {
return Message{}, io.EOF
}
// Skip anything before the first separator.
for r.next == "" {
line, _, err := r.readLine(r.keep())
if err != nil {
r.eof = true
if err == io.EOF {
return Message{}, io.EOF
}
return Message{}, err
}
if s := strings.TrimRight(string(line), "\r\n"); separator.MatchString(s) {
r.next = s
}
}
r.index++
msg := Message{Index: r.index, Envelope: r.next}
r.next = ""
var buf bytes.Buffer
oversized := false
size := 0
for {
line, n, err := r.readLine(r.keep())
if err != nil {
r.eof = true
if err != io.EOF {
return Message{}, err
}
break
}
if s := strings.TrimRight(string(line), "\r\n"); separator.MatchString(s) {
r.next = s
break
}
if oversized {
continue
}
if escaped.Match(line) {
line = line[1:]
n--
}
size += n
if r.max > 0 && size > r.max {
oversized = true
buf.Reset()
continue
}
buf.Write(line)
}
if oversized {
msg.Err = ErrTooLarge
return msg, ErrTooLarge
}
msg.Data = trimSeparatorLine(buf.Bytes())
return msg, nil
}
// trimSeparatorLine drops the one blank line that precedes a separator.
func trimSeparatorLine(b []byte) []byte {
switch {
case bytes.HasSuffix(b, []byte("\r\n\r\n")):
return b[:len(b)-2]
case bytes.HasSuffix(b, []byte("\n\n")):
return b[:len(b)-1]
}
return b
}