- readLine now holds at most max(limit+1, 1024) bytes of a line and reports its full length; the rest is read and thrown away, so a single 25 MiB line is no longer read whole. - The message size is counted from the reported line lengths (after restoring >From), so the limit is unchanged for normal input. - Tests: a line over the limit mid-file, at the end of the input and before the first separator, long lines under the limit, a tiny limit, exact boundary, readLine itself.
182 lines
4.5 KiB
Go
182 lines
4.5 KiB
Go
// Package mbox reads mbox archives, such as the ones Mailman exports.
|
|
//
|
|
// A message starts at a line like "From alice@example.com Tue Jan 2 15:04:05 2024".
|
|
// A line that merely begins with "From " in prose does not split a message. Lines
|
|
// escaped as ">From " are restored (mboxrd); the blank line before a separator
|
|
// belongs to the separator, not to the message.
|
|
package mbox
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"errors"
|
|
"io"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// ErrTooLarge is returned with a message that exceeded the size limit and was skipped.
|
|
var ErrTooLarge = errors.New("message too large")
|
|
|
|
var (
|
|
separator = regexp.MustCompile(`^From \S+\s+(Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2}\s+\d{1,2}:\d{2}`)
|
|
escaped = regexp.MustCompile(`^>+From `)
|
|
)
|
|
|
|
// Message is one message of the mbox.
|
|
type Message struct {
|
|
Index int // 1-based position in the file
|
|
Envelope string // the "From " line
|
|
Data []byte // the RFC 5322 message; nil when Err is set
|
|
Err error
|
|
}
|
|
|
|
// Date returns the date of the "From " line, when it has a parsable one.
|
|
func (m Message) Date() (time.Time, bool) {
|
|
f := strings.Fields(m.Envelope)
|
|
if len(f) < 3 || f[0] != "From" {
|
|
return time.Time{}, false
|
|
}
|
|
rest := strings.Join(f[2:], " ")
|
|
for _, layout := range []string{
|
|
"Mon Jan 2 15:04:05 2006",
|
|
"Mon Jan 2 15:04:05 -0700 2006",
|
|
"Mon Jan 2 15:04:05 MST 2006",
|
|
"Mon Jan 2 15:04 2006",
|
|
} {
|
|
if t, err := time.Parse(layout, rest); err == nil {
|
|
return t, true
|
|
}
|
|
}
|
|
return time.Time{}, false
|
|
}
|
|
|
|
// Reader returns the messages of an mbox one at a time.
|
|
type Reader struct {
|
|
br *bufio.Reader
|
|
max int
|
|
next string // envelope of the message that comes next, once seen
|
|
index int
|
|
eof bool
|
|
}
|
|
|
|
// NewReader reads from r. Messages larger than maxSize bytes (0 means no limit)
|
|
// are skipped and reported with ErrTooLarge.
|
|
func NewReader(r io.Reader, maxSize int) *Reader {
|
|
return &Reader{br: bufio.NewReaderSize(r, 64*1024), max: maxSize}
|
|
}
|
|
|
|
// minKeep is the least a line may be cut to, so a "From " separator is never cut short.
|
|
const minKeep = 1024
|
|
|
|
// keep is how many bytes of a line are worth holding: a message with a longer line is too
|
|
// large anyway, so the rest is read and thrown away instead of buffered.
|
|
func (r *Reader) keep() int {
|
|
if r.max <= 0 {
|
|
return 0 // no limit
|
|
}
|
|
return max(r.max+1, minKeep)
|
|
}
|
|
|
|
// readLine returns the next line including its newline, cut to keep bytes (0 means no cut),
|
|
// and the full length of the line. At the end of the input it returns the last partial line
|
|
// (if any) with io.EOF on the following call.
|
|
func (r *Reader) readLine(keep int) ([]byte, int, error) {
|
|
var line []byte
|
|
n := 0
|
|
for {
|
|
chunk, err := r.br.ReadSlice('\n')
|
|
n += len(chunk)
|
|
if room := keep - len(line); keep == 0 || room > 0 {
|
|
if keep > 0 && len(chunk) > room {
|
|
chunk = chunk[:room]
|
|
}
|
|
line = append(line, chunk...)
|
|
}
|
|
switch err {
|
|
case nil:
|
|
return line, n, nil
|
|
case bufio.ErrBufferFull:
|
|
continue
|
|
case io.EOF:
|
|
if n > 0 {
|
|
return line, n, nil
|
|
}
|
|
}
|
|
return line, n, err
|
|
}
|
|
}
|
|
|
|
// Next returns the next message, or io.EOF when there are no more.
|
|
func (r *Reader) Next() (Message, error) {
|
|
if r.eof {
|
|
return Message{}, io.EOF
|
|
}
|
|
// Skip anything before the first separator.
|
|
for r.next == "" {
|
|
line, _, err := r.readLine(r.keep())
|
|
if err != nil {
|
|
r.eof = true
|
|
if err == io.EOF {
|
|
return Message{}, io.EOF
|
|
}
|
|
return Message{}, err
|
|
}
|
|
if s := strings.TrimRight(string(line), "\r\n"); separator.MatchString(s) {
|
|
r.next = s
|
|
}
|
|
}
|
|
r.index++
|
|
msg := Message{Index: r.index, Envelope: r.next}
|
|
r.next = ""
|
|
|
|
var buf bytes.Buffer
|
|
oversized := false
|
|
size := 0
|
|
for {
|
|
line, n, err := r.readLine(r.keep())
|
|
if err != nil {
|
|
r.eof = true
|
|
if err != io.EOF {
|
|
return Message{}, err
|
|
}
|
|
break
|
|
}
|
|
if s := strings.TrimRight(string(line), "\r\n"); separator.MatchString(s) {
|
|
r.next = s
|
|
break
|
|
}
|
|
if oversized {
|
|
continue
|
|
}
|
|
if escaped.Match(line) {
|
|
line = line[1:]
|
|
n--
|
|
}
|
|
size += n
|
|
if r.max > 0 && size > r.max {
|
|
oversized = true
|
|
buf.Reset()
|
|
continue
|
|
}
|
|
buf.Write(line)
|
|
}
|
|
if oversized {
|
|
msg.Err = ErrTooLarge
|
|
return msg, ErrTooLarge
|
|
}
|
|
msg.Data = trimSeparatorLine(buf.Bytes())
|
|
return msg, nil
|
|
}
|
|
|
|
// trimSeparatorLine drops the one blank line that precedes a separator.
|
|
func trimSeparatorLine(b []byte) []byte {
|
|
switch {
|
|
case bytes.HasSuffix(b, []byte("\r\n\r\n")):
|
|
return b[:len(b)-2]
|
|
case bytes.HasSuffix(b, []byte("\n\n")):
|
|
return b[:len(b)-1]
|
|
}
|
|
return b
|
|
}
|