feat(mbox,store): mbox reader, batch archiving and thread rebuilding

- Add internal/mbox: streaming reader with strict From_ detection, mboxrd unescaping, size cap and envelope dates
- Add Store.ArchiveBatch: many messages in one transaction
- Add Store.Rethread: rebuild threads from In-Reply-To whatever the insertion order, tolerating cycles
This commit is contained in:
tgiachi 2026-11-09 17:13:56 +01:00
commit ba6c37bc21
4 changed files with 599 additions and 4 deletions

147
internal/mbox/mbox.go Normal file
View file

@ -0,0 +1,147 @@
// Package mbox reads mbox archives, such as the ones Mailman exports.
//
// A message starts at a line like "From alice@example.com Tue Jan 2 15:04:05 2024".
// A line that merely begins with "From " in prose does not split a message. Lines
// escaped as ">From " are restored (mboxrd); the blank line before a separator
// belongs to the separator, not to the message.
package mbox
import (
"bufio"
"bytes"
"errors"
"io"
"regexp"
"strings"
"time"
)
// ErrTooLarge is returned with a message that exceeded the size limit and was skipped.
var ErrTooLarge = errors.New("message too large")
var (
separator = regexp.MustCompile(`^From \S+\s+(Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2}\s+\d{1,2}:\d{2}`)
escaped = regexp.MustCompile(`^>+From `)
)
// Message is one message of the mbox.
type Message struct {
Index int // 1-based position in the file
Envelope string // the "From " line
Data []byte // the RFC 5322 message; nil when Err is set
Err error
}
// Date returns the date of the "From " line, when it has a parsable one.
func (m Message) Date() (time.Time, bool) {
f := strings.Fields(m.Envelope)
if len(f) < 3 || f[0] != "From" {
return time.Time{}, false
}
rest := strings.Join(f[2:], " ")
for _, layout := range []string{
"Mon Jan 2 15:04:05 2006",
"Mon Jan 2 15:04:05 -0700 2006",
"Mon Jan 2 15:04:05 MST 2006",
"Mon Jan 2 15:04 2006",
} {
if t, err := time.Parse(layout, rest); err == nil {
return t, true
}
}
return time.Time{}, false
}
// Reader returns the messages of an mbox one at a time.
type Reader struct {
br *bufio.Reader
max int
next string // envelope of the message that comes next, once seen
index int
eof bool
}
// NewReader reads from r. Messages larger than maxSize bytes (0 means no limit)
// are skipped and reported with ErrTooLarge.
func NewReader(r io.Reader, maxSize int) *Reader {
return &Reader{br: bufio.NewReaderSize(r, 64*1024), max: maxSize}
}
// readLine returns the next line including its newline. At the end of the input
// it returns the last partial line (if any) with io.EOF on the following call.
func (r *Reader) readLine() ([]byte, error) {
line, err := r.br.ReadBytes('\n')
if err == io.EOF && len(line) > 0 {
return line, nil
}
return line, err
}
// Next returns the next message, or io.EOF when there are no more.
func (r *Reader) Next() (Message, error) {
if r.eof {
return Message{}, io.EOF
}
// Skip anything before the first separator.
for r.next == "" {
line, err := r.readLine()
if err != nil {
r.eof = true
if err == io.EOF {
return Message{}, io.EOF
}
return Message{}, err
}
if s := strings.TrimRight(string(line), "\r\n"); separator.MatchString(s) {
r.next = s
}
}
r.index++
msg := Message{Index: r.index, Envelope: r.next}
r.next = ""
var buf bytes.Buffer
oversized := false
for {
line, err := r.readLine()
if err != nil {
r.eof = true
if err != io.EOF {
return Message{}, err
}
break
}
if s := strings.TrimRight(string(line), "\r\n"); separator.MatchString(s) {
r.next = s
break
}
if oversized {
continue
}
if escaped.Match(line) {
line = line[1:]
}
buf.Write(line)
if r.max > 0 && buf.Len() > r.max {
oversized = true
buf.Reset()
}
}
if oversized {
msg.Err = ErrTooLarge
return msg, ErrTooLarge
}
msg.Data = trimSeparatorLine(buf.Bytes())
return msg, nil
}
// trimSeparatorLine drops the one blank line that precedes a separator.
func trimSeparatorLine(b []byte) []byte {
switch {
case bytes.HasSuffix(b, []byte("\r\n\r\n")):
return b[:len(b)-2]
case bytes.HasSuffix(b, []byte("\n\n")):
return b[:len(b)-1]
}
return b
}

165
internal/mbox/mbox_test.go Normal file
View file

@ -0,0 +1,165 @@
package mbox
import (
"errors"
"io"
"strings"
"testing"
"time"
)
func readAll(t *testing.T, input string, maxSize int) []Message {
t.Helper()
r := NewReader(strings.NewReader(input), maxSize)
var out []Message
for {
m, err := r.Next()
if err == io.EOF {
return out
}
if err != nil && !errors.Is(err, ErrTooLarge) {
t.Fatalf("next: %v", err)
}
m.Err = err
out = append(out, m)
}
}
const two = "From alice@example.com Tue Jan 2 15:04:05 2024\n" +
"From: alice@example.com\nSubject: one\n\nfirst body\n\n" +
"From bob@example.com Wed Jan 3 09:00:00 2024\n" +
"From: bob@example.com\nSubject: two\n\nsecond body\n"
func TestSplitsMessages(t *testing.T) {
ms := readAll(t, two, 0)
if len(ms) != 2 {
t.Fatalf("got %d messages", len(ms))
}
if string(ms[0].Data) != "From: alice@example.com\nSubject: one\n\nfirst body\n" {
t.Fatalf("first = %q", ms[0].Data)
}
if string(ms[1].Data) != "From: bob@example.com\nSubject: two\n\nsecond body\n" {
t.Fatalf("second = %q", ms[1].Data)
}
if ms[0].Envelope != "From alice@example.com Tue Jan 2 15:04:05 2024" {
t.Fatalf("envelope = %q", ms[0].Envelope)
}
}
func TestEnvelopeDate(t *testing.T) {
ms := readAll(t, two, 0)
want := time.Date(2024, 1, 2, 15, 4, 5, 0, time.UTC)
if got, ok := ms[0].Date(); !ok || !got.Equal(want) {
t.Fatalf("date = %v %v", got, ok)
}
for _, env := range []string{
"From x@y.z Tue Jan 2 15:04:05 +0200 2024",
"From x@y.z Tue Jan 2 15:04:05 2024",
"From x@y.z Tue Jan 2 15:04 2024",
"From MAILER-DAEMON Tue Jan 2 15:04:05 2024",
} {
if _, ok := (Message{Envelope: env}).Date(); !ok {
t.Errorf("%q must parse", env)
}
}
for _, env := range []string{"", "From x@y.z", "From x@y.z not a date"} {
if _, ok := (Message{Envelope: env}).Date(); ok {
t.Errorf("%q must not parse", env)
}
}
}
func TestProseLineStartingWithFromDoesNotSplit(t *testing.T) {
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nFrom now on we meet on Fridays.\nFrom: not a separator either\n"
ms := readAll(t, in, 0)
if len(ms) != 1 || !strings.Contains(string(ms[0].Data), "From now on we meet on Fridays.") {
t.Fatalf("messages: %d %q", len(ms), ms[0].Data)
}
}
func TestMboxrdUnescaping(t *testing.T) {
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n>From the log:\n>>From a quoted line\n> From not escaped\nplain\n"
ms := readAll(t, in, 0)
body := string(ms[0].Data)
if !strings.Contains(body, "\nFrom the log:\n") || !strings.Contains(body, "\n>From a quoted line\n") || !strings.Contains(body, "\n> From not escaped\n") {
t.Fatalf("body = %q", body)
}
}
func TestLeadingJunkAndEmptyInput(t *testing.T) {
if got := readAll(t, "", 0); len(got) != 0 {
t.Fatalf("empty input: %d", len(got))
}
if got := readAll(t, "\n\nsome preamble\nFrom a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nbody\n", 0); len(got) != 1 {
t.Fatalf("preamble must be ignored: %d", len(got))
}
if got := readAll(t, "just text, no separator at all\n", 0); len(got) != 0 {
t.Fatalf("no messages expected: %d", len(got))
}
}
func TestCRLFAndMissingFinalNewline(t *testing.T) {
in := "From a@x.com Tue Jan 2 15:04:05 2024\r\nSubject: s\r\n\r\nbody without newline"
ms := readAll(t, in, 0)
if len(ms) != 1 || !strings.HasSuffix(string(ms[0].Data), "body without newline") || !strings.Contains(string(ms[0].Data), "Subject: s\r\n") {
t.Fatalf("%q", ms[0].Data)
}
crlf := "From a@x.com Tue Jan 2 15:04:05 2024\r\nSubject: one\r\n\r\nb\r\n\r\nFrom b@x.com Tue Jan 2 15:04:05 2024\r\nSubject: two\r\n\r\nb2\r\n"
if got := readAll(t, crlf, 0); len(got) != 2 {
t.Fatalf("CRLF separators: %d", len(got))
}
}
func TestVeryLongLinesAreFine(t *testing.T) {
long := strings.Repeat("x", 300_000)
ms := readAll(t, "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n"+long+"\n", 0)
if len(ms) != 1 || !strings.Contains(string(ms[0].Data), long) {
t.Fatal("a long line must survive")
}
}
func TestOversizedMessagesAreReportedAndSkipped(t *testing.T) {
big := strings.Repeat("y\n", 5000)
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: ok1\n\nsmall\n" +
"From b@x.com Tue Jan 2 15:04:05 2024\nSubject: big\n\n" + big +
"From c@x.com Tue Jan 2 15:04:05 2024\nSubject: ok2\n\nsmall\n"
ms := readAll(t, in, 1000)
if len(ms) != 3 {
t.Fatalf("got %d", len(ms))
}
if ms[0].Err != nil || ms[2].Err != nil {
t.Fatalf("small messages must pass: %v %v", ms[0].Err, ms[2].Err)
}
if !errors.Is(ms[1].Err, ErrTooLarge) || ms[1].Data != nil {
t.Fatalf("big message: %v %d bytes", ms[1].Err, len(ms[1].Data))
}
if ms[1].Envelope != "From b@x.com Tue Jan 2 15:04:05 2024" || !strings.Contains(string(ms[2].Data), "ok2") {
t.Fatalf("the reader must resynchronise after a big message: %+v", ms[2])
}
}
func TestIndexIsOneBased(t *testing.T) {
ms := readAll(t, two, 0)
if ms[0].Index != 1 || ms[1].Index != 2 {
t.Fatalf("indexes: %d %d", ms[0].Index, ms[1].Index)
}
}
func TestReadErrorsArePropagated(t *testing.T) {
r := NewReader(io.MultiReader(strings.NewReader("From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nbody\n"), errReader{}), 0)
var last error
for i := 0; i < 5; i++ {
_, err := r.Next()
if err != nil {
last = err
break
}
}
if last == nil || last == io.EOF {
t.Fatalf("want the read error, got %v", last)
}
}
type errReader struct{}
func (errReader) Read([]byte) (int, error) { return 0, errors.New("disk on fire") }

View file

@ -227,3 +227,159 @@ func TestLongBodyIsStoredAsIs(t *testing.T) {
t.Fatalf("body len %d", len(got.Body))
}
}
func TestArchiveBatchInsertsDeduplicatesAndThreads(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
s.ArchiveMessage(l.ID, msg("<old@x>", "Already here", "x"))
reply := msg("<r@x>", "Re: Topic", "reply")
reply.InReplyTo = "<root@x>"
res, err := s.ArchiveBatch(l.ID, []Message{
msg("<root@x>", "Topic", "root"),
reply,
msg("<old@x>", "Already here", "x"), // duplicate of an existing message
msg("<root@x>", "Topic", "root"), // duplicate inside the batch
})
if err != nil {
t.Fatal(err)
}
if len(res) != 4 || !res[0].Created || !res[1].Created || res[2].Created || res[3].Created {
t.Fatalf("results: %+v", res)
}
if res[3].ID != res[0].ID {
t.Fatalf("a duplicate reports the id of the stored message: %+v", res)
}
r, _ := s.GetMessage(res[1].ID)
if r.ThreadID != res[0].ID {
t.Fatalf("threading inside a batch: thread %d, root %d", r.ThreadID, res[0].ID)
}
if _, total, _ := s.ListMessages(l.ID, 10, 0); total != 3 {
t.Fatalf("total = %d", total)
}
if got, _, _ := s.SearchMessages(l.ID, "reply", 10, 0); len(got) != 1 {
t.Fatal("batch inserts must be searchable")
}
}
func TestArchiveBatchIsAtomic(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
_, err := s.ArchiveBatch(l.ID, []Message{msg("<a@x>", "A", "a"), {Subject: "no id"}})
if err == nil {
t.Fatal("a message without an id must fail the batch")
}
if _, total, _ := s.ListMessages(l.ID, 10, 0); total != 0 {
t.Fatalf("nothing may be stored after a failed batch, got %d", total)
}
if res, err := s.ArchiveBatch(l.ID, nil); err != nil || len(res) != 0 {
t.Fatalf("empty batch: %v %v", res, err)
}
}
func threadOf(t *testing.T, s *Store, id int64) int64 {
t.Helper()
m, err := s.GetMessage(id)
if err != nil {
t.Fatal(err)
}
return m.ThreadID
}
func TestRethreadFixesMessagesImportedOutOfOrder(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
// Children first, root last: at insertion time none of the parents exists.
c2 := msg("<c2@x>", "Re: Re: T", "c2")
c2.InReplyTo = "<c1@x>"
c1 := msg("<c1@x>", "Re: T", "c1")
c1.InReplyTo = "<root@x>"
id2, _, _ := s.ArchiveMessage(l.ID, c2)
id1, _, _ := s.ArchiveMessage(l.ID, c1)
root, _, _ := s.ArchiveMessage(l.ID, msg("<root@x>", "T", "root"))
other, _, _ := s.ArchiveMessage(l.ID, msg("<other@x>", "Other", "o"))
if threadOf(t, s, id1) == root {
t.Fatal("precondition: the thread is broken before rethreading")
}
changed, err := s.Rethread(l.ID)
if err != nil {
t.Fatal(err)
}
if changed != 2 {
t.Fatalf("changed = %d, want the 2 broken messages", changed)
}
for _, id := range []int64{id1, id2, root} {
if got := threadOf(t, s, id); got != root {
t.Fatalf("message %d thread = %d, want %d", id, got, root)
}
}
if threadOf(t, s, other) != other {
t.Fatal("an unrelated message keeps its own thread")
}
if again, _ := s.Rethread(l.ID); again != 0 {
t.Fatalf("rethreading twice changes nothing, got %d", again)
}
if th, _ := s.ThreadMessages(l.ID, root); len(th) != 3 {
t.Fatalf("thread has %d messages", len(th))
}
}
func TestRethreadSurvivesCyclesAndSelfReplies(t *testing.T) {
s := newTestStore(t)
l, _ := s.CreateList("dev", "example.org", "Dev")
a := msg("<a@x>", "A", "a")
a.InReplyTo = "<b@x>"
b := msg("<b@x>", "B", "b")
b.InReplyTo = "<a@x>"
self := msg("<s@x>", "Self", "s")
self.InReplyTo = "<s@x>"
ia, _, _ := s.ArchiveMessage(l.ID, a)
ib, _, _ := s.ArchiveMessage(l.ID, b)
is, _, _ := s.ArchiveMessage(l.ID, self)
done := make(chan error, 1)
go func() { _, err := s.Rethread(l.ID); done <- err }()
select {
case err := <-done:
if err != nil {
t.Fatal(err)
}
case <-time.After(3 * time.Second):
t.Fatal("Rethread looped forever on a cycle")
}
if threadOf(t, s, ia) != threadOf(t, s, ib) {
t.Fatal("the two messages of a cycle share a thread")
}
if got := threadOf(t, s, ia); got != ia {
t.Fatalf("the cycle's thread is its smallest id (%d), got %d", ia, got)
}
if threadOf(t, s, is) != is {
t.Fatal("a message that replies to itself is its own thread")
}
}
func TestRethreadOnlyTouchesTheGivenList(t *testing.T) {
s := newTestStore(t)
a, _ := s.CreateList("a", "example.org", "A")
b, _ := s.CreateList("b", "example.org", "B")
child := msg("<c@x>", "Re", "c")
child.InReplyTo = "<p@x>"
cb, _, _ := s.ArchiveMessage(b.ID, child)
s.ArchiveMessage(b.ID, msg("<p@x>", "P", "p"))
// A parent with the same Message-ID in another list must not be used.
s.ArchiveMessage(a.ID, msg("<p@x>", "P in a", "p"))
if n, _ := s.Rethread(a.ID); n != 0 {
t.Fatalf("list a has nothing to fix, changed %d", n)
}
if threadOf(t, s, cb) != cb {
t.Fatal("list b must be untouched by rethreading list a")
}
if n, _ := s.Rethread(b.ID); n != 1 {
t.Fatalf("list b: changed %d", n)
}
if threadOf(t, s, cb) == cb {
t.Fatal("list b's child must now follow its parent")
}
}

View file

@ -590,14 +590,52 @@ func scanMessage(row interface{ Scan(...any) error }, withBody bool) (Message, e
return m, err
}
// querier is what archiveOne needs from a *sql.DB or a *sql.Tx.
type querier interface {
Exec(query string, args ...any) (sql.Result, error)
QueryRow(query string, args ...any) *sql.Row
}
// ArchiveResult is the outcome for one message of ArchiveBatch.
type ArchiveResult struct {
ID int64
Created bool
}
// ArchiveMessage stores a post. A message already archived in the same list
// (same Message-ID) is not stored again: created is false and the existing id
// is returned, so MTA retries are harmless.
func (s *Store) ArchiveMessage(listID int64, m Message) (id int64, created bool, err error) {
return archiveOne(s.db, listID, m)
}
// ArchiveBatch stores many messages in one transaction: all or none. It is much
// faster than ArchiveMessage in a loop, because SQLite syncs once per commit.
func (s *Store) ArchiveBatch(listID int64, ms []Message) ([]ArchiveResult, error) {
if len(ms) == 0 {
return nil, nil
}
tx, err := s.db.Begin()
if err != nil {
return nil, err
}
defer tx.Rollback()
out := make([]ArchiveResult, 0, len(ms))
for _, m := range ms {
id, created, err := archiveOne(tx, listID, m)
if err != nil {
return nil, err
}
out = append(out, ArchiveResult{ID: id, Created: created})
}
return out, tx.Commit()
}
func archiveOne(q querier, listID int64, m Message) (id int64, created bool, err error) {
if m.MessageID == "" {
return 0, false, errors.New("message id is required")
}
err = s.db.QueryRow(`SELECT id FROM archive WHERE list_id=? AND message_id=?`, listID, m.MessageID).Scan(&id)
err = q.QueryRow(`SELECT id FROM archive WHERE list_id=? AND message_id=?`, listID, m.MessageID).Scan(&id)
if err == nil {
return id, false, nil
}
@ -606,12 +644,12 @@ func (s *Store) ArchiveMessage(listID int64, m Message) (id int64, created bool,
}
var thread int64
if m.InReplyTo != "" {
e := s.db.QueryRow(`SELECT thread_id FROM archive WHERE list_id=? AND message_id=?`, listID, m.InReplyTo).Scan(&thread)
e := q.QueryRow(`SELECT thread_id FROM archive WHERE list_id=? AND message_id=?`, listID, m.InReplyTo).Scan(&thread)
if e != nil && !errors.Is(e, sql.ErrNoRows) {
return 0, false, e
}
}
res, err := s.db.Exec(`INSERT INTO archive(list_id, thread_id, message_id, in_reply_to, subject, sender, date, body, size)
res, err := q.Exec(`INSERT INTO archive(list_id, thread_id, message_id, in_reply_to, subject, sender, date, body, size)
VALUES(?,?,?,?,?,?,?,?,?)`,
listID, thread, m.MessageID, m.InReplyTo, m.Subject, m.Sender, m.Date.Unix(), m.Body, len(m.Body))
if err != nil {
@ -619,13 +657,102 @@ func (s *Store) ArchiveMessage(listID int64, m Message) (id int64, created bool,
}
id, _ = res.LastInsertId()
if thread == 0 {
if _, err := s.db.Exec(`UPDATE archive SET thread_id=id WHERE id=?`, id); err != nil {
if _, err := q.Exec(`UPDATE archive SET thread_id=id WHERE id=?`, id); err != nil {
return 0, false, err
}
}
return id, true, nil
}
// Rethread recomputes the thread of every message of a list from the In-Reply-To
// links, and returns how many messages changed thread. Insertion order does not
// matter: a message imported before its parent ends up in the parent's thread.
// A cycle of replies becomes one thread whose id is the smallest id in the cycle.
func (s *Store) Rethread(listID int64) (int, error) {
rows, err := s.db.Query(`SELECT id, message_id, in_reply_to, thread_id FROM archive WHERE list_id=?`, listID)
if err != nil {
return 0, err
}
type row struct {
id, thread int64
replyTo string
}
var all []row
byMsgID := map[string]int64{}
for rows.Next() {
var r row
var msgID string
if err := rows.Scan(&r.id, &msgID, &r.replyTo, &r.thread); err != nil {
rows.Close()
return 0, err
}
all = append(all, r)
byMsgID[msgID] = r.id
}
rows.Close()
if err := rows.Err(); err != nil {
return 0, err
}
parent := map[int64]int64{}
for _, r := range all {
if p, ok := byMsgID[r.replyTo]; ok && r.replyTo != "" {
parent[r.id] = p
}
}
roots := map[int64]int64{}
root := func(start int64) int64 {
if r, ok := roots[start]; ok {
return r
}
path := []int64{start}
seen := map[int64]int{start: 0}
cur := start
var result int64
for {
p, ok := parent[cur]
if !ok {
result = cur
break
}
if r, known := roots[p]; known {
result = r
break
}
if at, loop := seen[p]; loop { // a cycle: its smallest id is the root
result = path[at]
for _, id := range path[at:] {
result = min(result, id)
}
break
}
seen[p] = len(path)
path = append(path, p)
cur = p
}
for _, id := range path {
roots[id] = result
}
return result
}
tx, err := s.db.Begin()
if err != nil {
return 0, err
}
defer tx.Rollback()
changed := 0
for _, r := range all {
if want := root(r.id); want != r.thread {
if _, err := tx.Exec(`UPDATE archive SET thread_id=? WHERE id=?`, want, r.id); err != nil {
return 0, err
}
changed++
}
}
return changed, tx.Commit()
}
// GetMessage returns an archived message with its body.
func (s *Store) GetMessage(id int64) (Message, error) {
return scanMessage(s.db.QueryRow(`SELECT `+msgCols+`, body FROM archive WHERE id=?`, id), true)