diff --git a/internal/mbox/mbox.go b/internal/mbox/mbox.go new file mode 100644 index 0000000..a6aa4b6 --- /dev/null +++ b/internal/mbox/mbox.go @@ -0,0 +1,147 @@ +// Package mbox reads mbox archives, such as the ones Mailman exports. +// +// A message starts at a line like "From alice@example.com Tue Jan 2 15:04:05 2024". +// A line that merely begins with "From " in prose does not split a message. Lines +// escaped as ">From " are restored (mboxrd); the blank line before a separator +// belongs to the separator, not to the message. +package mbox + +import ( + "bufio" + "bytes" + "errors" + "io" + "regexp" + "strings" + "time" +) + +// ErrTooLarge is returned with a message that exceeded the size limit and was skipped. +var ErrTooLarge = errors.New("message too large") + +var ( + separator = regexp.MustCompile(`^From \S+\s+(Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2}\s+\d{1,2}:\d{2}`) + escaped = regexp.MustCompile(`^>+From `) +) + +// Message is one message of the mbox. +type Message struct { + Index int // 1-based position in the file + Envelope string // the "From " line + Data []byte // the RFC 5322 message; nil when Err is set + Err error +} + +// Date returns the date of the "From " line, when it has a parsable one. +func (m Message) Date() (time.Time, bool) { + f := strings.Fields(m.Envelope) + if len(f) < 3 || f[0] != "From" { + return time.Time{}, false + } + rest := strings.Join(f[2:], " ") + for _, layout := range []string{ + "Mon Jan 2 15:04:05 2006", + "Mon Jan 2 15:04:05 -0700 2006", + "Mon Jan 2 15:04:05 MST 2006", + "Mon Jan 2 15:04 2006", + } { + if t, err := time.Parse(layout, rest); err == nil { + return t, true + } + } + return time.Time{}, false +} + +// Reader returns the messages of an mbox one at a time. +type Reader struct { + br *bufio.Reader + max int + next string // envelope of the message that comes next, once seen + index int + eof bool +} + +// NewReader reads from r. Messages larger than maxSize bytes (0 means no limit) +// are skipped and reported with ErrTooLarge. +func NewReader(r io.Reader, maxSize int) *Reader { + return &Reader{br: bufio.NewReaderSize(r, 64*1024), max: maxSize} +} + +// readLine returns the next line including its newline. At the end of the input +// it returns the last partial line (if any) with io.EOF on the following call. +func (r *Reader) readLine() ([]byte, error) { + line, err := r.br.ReadBytes('\n') + if err == io.EOF && len(line) > 0 { + return line, nil + } + return line, err +} + +// Next returns the next message, or io.EOF when there are no more. +func (r *Reader) Next() (Message, error) { + if r.eof { + return Message{}, io.EOF + } + // Skip anything before the first separator. + for r.next == "" { + line, err := r.readLine() + if err != nil { + r.eof = true + if err == io.EOF { + return Message{}, io.EOF + } + return Message{}, err + } + if s := strings.TrimRight(string(line), "\r\n"); separator.MatchString(s) { + r.next = s + } + } + r.index++ + msg := Message{Index: r.index, Envelope: r.next} + r.next = "" + + var buf bytes.Buffer + oversized := false + for { + line, err := r.readLine() + if err != nil { + r.eof = true + if err != io.EOF { + return Message{}, err + } + break + } + if s := strings.TrimRight(string(line), "\r\n"); separator.MatchString(s) { + r.next = s + break + } + if oversized { + continue + } + if escaped.Match(line) { + line = line[1:] + } + buf.Write(line) + if r.max > 0 && buf.Len() > r.max { + oversized = true + buf.Reset() + } + } + if oversized { + msg.Err = ErrTooLarge + return msg, ErrTooLarge + } + msg.Data = trimSeparatorLine(buf.Bytes()) + return msg, nil +} + +// trimSeparatorLine drops the one blank line that precedes a separator. +func trimSeparatorLine(b []byte) []byte { + switch { + case bytes.HasSuffix(b, []byte("\r\n\r\n")): + return b[:len(b)-2] + case bytes.HasSuffix(b, []byte("\n\n")): + return b[:len(b)-1] + } + return b +} diff --git a/internal/mbox/mbox_test.go b/internal/mbox/mbox_test.go new file mode 100644 index 0000000..c508f43 --- /dev/null +++ b/internal/mbox/mbox_test.go @@ -0,0 +1,165 @@ +package mbox + +import ( + "errors" + "io" + "strings" + "testing" + "time" +) + +func readAll(t *testing.T, input string, maxSize int) []Message { + t.Helper() + r := NewReader(strings.NewReader(input), maxSize) + var out []Message + for { + m, err := r.Next() + if err == io.EOF { + return out + } + if err != nil && !errors.Is(err, ErrTooLarge) { + t.Fatalf("next: %v", err) + } + m.Err = err + out = append(out, m) + } +} + +const two = "From alice@example.com Tue Jan 2 15:04:05 2024\n" + + "From: alice@example.com\nSubject: one\n\nfirst body\n\n" + + "From bob@example.com Wed Jan 3 09:00:00 2024\n" + + "From: bob@example.com\nSubject: two\n\nsecond body\n" + +func TestSplitsMessages(t *testing.T) { + ms := readAll(t, two, 0) + if len(ms) != 2 { + t.Fatalf("got %d messages", len(ms)) + } + if string(ms[0].Data) != "From: alice@example.com\nSubject: one\n\nfirst body\n" { + t.Fatalf("first = %q", ms[0].Data) + } + if string(ms[1].Data) != "From: bob@example.com\nSubject: two\n\nsecond body\n" { + t.Fatalf("second = %q", ms[1].Data) + } + if ms[0].Envelope != "From alice@example.com Tue Jan 2 15:04:05 2024" { + t.Fatalf("envelope = %q", ms[0].Envelope) + } +} + +func TestEnvelopeDate(t *testing.T) { + ms := readAll(t, two, 0) + want := time.Date(2024, 1, 2, 15, 4, 5, 0, time.UTC) + if got, ok := ms[0].Date(); !ok || !got.Equal(want) { + t.Fatalf("date = %v %v", got, ok) + } + for _, env := range []string{ + "From x@y.z Tue Jan 2 15:04:05 +0200 2024", + "From x@y.z Tue Jan 2 15:04:05 2024", + "From x@y.z Tue Jan 2 15:04 2024", + "From MAILER-DAEMON Tue Jan 2 15:04:05 2024", + } { + if _, ok := (Message{Envelope: env}).Date(); !ok { + t.Errorf("%q must parse", env) + } + } + for _, env := range []string{"", "From x@y.z", "From x@y.z not a date"} { + if _, ok := (Message{Envelope: env}).Date(); ok { + t.Errorf("%q must not parse", env) + } + } +} + +func TestProseLineStartingWithFromDoesNotSplit(t *testing.T) { + in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nFrom now on we meet on Fridays.\nFrom: not a separator either\n" + ms := readAll(t, in, 0) + if len(ms) != 1 || !strings.Contains(string(ms[0].Data), "From now on we meet on Fridays.") { + t.Fatalf("messages: %d %q", len(ms), ms[0].Data) + } +} + +func TestMboxrdUnescaping(t *testing.T) { + in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n>From the log:\n>>From a quoted line\n> From not escaped\nplain\n" + ms := readAll(t, in, 0) + body := string(ms[0].Data) + if !strings.Contains(body, "\nFrom the log:\n") || !strings.Contains(body, "\n>From a quoted line\n") || !strings.Contains(body, "\n> From not escaped\n") { + t.Fatalf("body = %q", body) + } +} + +func TestLeadingJunkAndEmptyInput(t *testing.T) { + if got := readAll(t, "", 0); len(got) != 0 { + t.Fatalf("empty input: %d", len(got)) + } + if got := readAll(t, "\n\nsome preamble\nFrom a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nbody\n", 0); len(got) != 1 { + t.Fatalf("preamble must be ignored: %d", len(got)) + } + if got := readAll(t, "just text, no separator at all\n", 0); len(got) != 0 { + t.Fatalf("no messages expected: %d", len(got)) + } +} + +func TestCRLFAndMissingFinalNewline(t *testing.T) { + in := "From a@x.com Tue Jan 2 15:04:05 2024\r\nSubject: s\r\n\r\nbody without newline" + ms := readAll(t, in, 0) + if len(ms) != 1 || !strings.HasSuffix(string(ms[0].Data), "body without newline") || !strings.Contains(string(ms[0].Data), "Subject: s\r\n") { + t.Fatalf("%q", ms[0].Data) + } + crlf := "From a@x.com Tue Jan 2 15:04:05 2024\r\nSubject: one\r\n\r\nb\r\n\r\nFrom b@x.com Tue Jan 2 15:04:05 2024\r\nSubject: two\r\n\r\nb2\r\n" + if got := readAll(t, crlf, 0); len(got) != 2 { + t.Fatalf("CRLF separators: %d", len(got)) + } +} + +func TestVeryLongLinesAreFine(t *testing.T) { + long := strings.Repeat("x", 300_000) + ms := readAll(t, "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n"+long+"\n", 0) + if len(ms) != 1 || !strings.Contains(string(ms[0].Data), long) { + t.Fatal("a long line must survive") + } +} + +func TestOversizedMessagesAreReportedAndSkipped(t *testing.T) { + big := strings.Repeat("y\n", 5000) + in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: ok1\n\nsmall\n" + + "From b@x.com Tue Jan 2 15:04:05 2024\nSubject: big\n\n" + big + + "From c@x.com Tue Jan 2 15:04:05 2024\nSubject: ok2\n\nsmall\n" + ms := readAll(t, in, 1000) + if len(ms) != 3 { + t.Fatalf("got %d", len(ms)) + } + if ms[0].Err != nil || ms[2].Err != nil { + t.Fatalf("small messages must pass: %v %v", ms[0].Err, ms[2].Err) + } + if !errors.Is(ms[1].Err, ErrTooLarge) || ms[1].Data != nil { + t.Fatalf("big message: %v %d bytes", ms[1].Err, len(ms[1].Data)) + } + if ms[1].Envelope != "From b@x.com Tue Jan 2 15:04:05 2024" || !strings.Contains(string(ms[2].Data), "ok2") { + t.Fatalf("the reader must resynchronise after a big message: %+v", ms[2]) + } +} + +func TestIndexIsOneBased(t *testing.T) { + ms := readAll(t, two, 0) + if ms[0].Index != 1 || ms[1].Index != 2 { + t.Fatalf("indexes: %d %d", ms[0].Index, ms[1].Index) + } +} + +func TestReadErrorsArePropagated(t *testing.T) { + r := NewReader(io.MultiReader(strings.NewReader("From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nbody\n"), errReader{}), 0) + var last error + for i := 0; i < 5; i++ { + _, err := r.Next() + if err != nil { + last = err + break + } + } + if last == nil || last == io.EOF { + t.Fatalf("want the read error, got %v", last) + } +} + +type errReader struct{} + +func (errReader) Read([]byte) (int, error) { return 0, errors.New("disk on fire") } diff --git a/internal/store/archive_test.go b/internal/store/archive_test.go index 7fe6b01..1ee9133 100644 --- a/internal/store/archive_test.go +++ b/internal/store/archive_test.go @@ -227,3 +227,159 @@ func TestLongBodyIsStoredAsIs(t *testing.T) { t.Fatalf("body len %d", len(got.Body)) } } + +func TestArchiveBatchInsertsDeduplicatesAndThreads(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.ArchiveMessage(l.ID, msg("", "Already here", "x")) + + reply := msg("", "Re: Topic", "reply") + reply.InReplyTo = "" + res, err := s.ArchiveBatch(l.ID, []Message{ + msg("", "Topic", "root"), + reply, + msg("", "Already here", "x"), // duplicate of an existing message + msg("", "Topic", "root"), // duplicate inside the batch + }) + if err != nil { + t.Fatal(err) + } + if len(res) != 4 || !res[0].Created || !res[1].Created || res[2].Created || res[3].Created { + t.Fatalf("results: %+v", res) + } + if res[3].ID != res[0].ID { + t.Fatalf("a duplicate reports the id of the stored message: %+v", res) + } + r, _ := s.GetMessage(res[1].ID) + if r.ThreadID != res[0].ID { + t.Fatalf("threading inside a batch: thread %d, root %d", r.ThreadID, res[0].ID) + } + if _, total, _ := s.ListMessages(l.ID, 10, 0); total != 3 { + t.Fatalf("total = %d", total) + } + if got, _, _ := s.SearchMessages(l.ID, "reply", 10, 0); len(got) != 1 { + t.Fatal("batch inserts must be searchable") + } +} + +func TestArchiveBatchIsAtomic(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + _, err := s.ArchiveBatch(l.ID, []Message{msg("", "A", "a"), {Subject: "no id"}}) + if err == nil { + t.Fatal("a message without an id must fail the batch") + } + if _, total, _ := s.ListMessages(l.ID, 10, 0); total != 0 { + t.Fatalf("nothing may be stored after a failed batch, got %d", total) + } + if res, err := s.ArchiveBatch(l.ID, nil); err != nil || len(res) != 0 { + t.Fatalf("empty batch: %v %v", res, err) + } +} + +func threadOf(t *testing.T, s *Store, id int64) int64 { + t.Helper() + m, err := s.GetMessage(id) + if err != nil { + t.Fatal(err) + } + return m.ThreadID +} + +func TestRethreadFixesMessagesImportedOutOfOrder(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + // Children first, root last: at insertion time none of the parents exists. + c2 := msg("", "Re: Re: T", "c2") + c2.InReplyTo = "" + c1 := msg("", "Re: T", "c1") + c1.InReplyTo = "" + id2, _, _ := s.ArchiveMessage(l.ID, c2) + id1, _, _ := s.ArchiveMessage(l.ID, c1) + root, _, _ := s.ArchiveMessage(l.ID, msg("", "T", "root")) + other, _, _ := s.ArchiveMessage(l.ID, msg("", "Other", "o")) + if threadOf(t, s, id1) == root { + t.Fatal("precondition: the thread is broken before rethreading") + } + + changed, err := s.Rethread(l.ID) + if err != nil { + t.Fatal(err) + } + if changed != 2 { + t.Fatalf("changed = %d, want the 2 broken messages", changed) + } + for _, id := range []int64{id1, id2, root} { + if got := threadOf(t, s, id); got != root { + t.Fatalf("message %d thread = %d, want %d", id, got, root) + } + } + if threadOf(t, s, other) != other { + t.Fatal("an unrelated message keeps its own thread") + } + if again, _ := s.Rethread(l.ID); again != 0 { + t.Fatalf("rethreading twice changes nothing, got %d", again) + } + if th, _ := s.ThreadMessages(l.ID, root); len(th) != 3 { + t.Fatalf("thread has %d messages", len(th)) + } +} + +func TestRethreadSurvivesCyclesAndSelfReplies(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + a := msg("", "A", "a") + a.InReplyTo = "" + b := msg("", "B", "b") + b.InReplyTo = "" + self := msg("", "Self", "s") + self.InReplyTo = "" + ia, _, _ := s.ArchiveMessage(l.ID, a) + ib, _, _ := s.ArchiveMessage(l.ID, b) + is, _, _ := s.ArchiveMessage(l.ID, self) + + done := make(chan error, 1) + go func() { _, err := s.Rethread(l.ID); done <- err }() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(3 * time.Second): + t.Fatal("Rethread looped forever on a cycle") + } + if threadOf(t, s, ia) != threadOf(t, s, ib) { + t.Fatal("the two messages of a cycle share a thread") + } + if got := threadOf(t, s, ia); got != ia { + t.Fatalf("the cycle's thread is its smallest id (%d), got %d", ia, got) + } + if threadOf(t, s, is) != is { + t.Fatal("a message that replies to itself is its own thread") + } +} + +func TestRethreadOnlyTouchesTheGivenList(t *testing.T) { + s := newTestStore(t) + a, _ := s.CreateList("a", "example.org", "A") + b, _ := s.CreateList("b", "example.org", "B") + child := msg("", "Re", "c") + child.InReplyTo = "" + cb, _, _ := s.ArchiveMessage(b.ID, child) + s.ArchiveMessage(b.ID, msg("", "P", "p")) + // A parent with the same Message-ID in another list must not be used. + s.ArchiveMessage(a.ID, msg("", "P in a", "p")) + + if n, _ := s.Rethread(a.ID); n != 0 { + t.Fatalf("list a has nothing to fix, changed %d", n) + } + if threadOf(t, s, cb) != cb { + t.Fatal("list b must be untouched by rethreading list a") + } + if n, _ := s.Rethread(b.ID); n != 1 { + t.Fatalf("list b: changed %d", n) + } + if threadOf(t, s, cb) == cb { + t.Fatal("list b's child must now follow its parent") + } +} diff --git a/internal/store/store.go b/internal/store/store.go index ec595f1..b14b2b0 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -590,14 +590,52 @@ func scanMessage(row interface{ Scan(...any) error }, withBody bool) (Message, e return m, err } +// querier is what archiveOne needs from a *sql.DB or a *sql.Tx. +type querier interface { + Exec(query string, args ...any) (sql.Result, error) + QueryRow(query string, args ...any) *sql.Row +} + +// ArchiveResult is the outcome for one message of ArchiveBatch. +type ArchiveResult struct { + ID int64 + Created bool +} + // ArchiveMessage stores a post. A message already archived in the same list // (same Message-ID) is not stored again: created is false and the existing id // is returned, so MTA retries are harmless. func (s *Store) ArchiveMessage(listID int64, m Message) (id int64, created bool, err error) { + return archiveOne(s.db, listID, m) +} + +// ArchiveBatch stores many messages in one transaction: all or none. It is much +// faster than ArchiveMessage in a loop, because SQLite syncs once per commit. +func (s *Store) ArchiveBatch(listID int64, ms []Message) ([]ArchiveResult, error) { + if len(ms) == 0 { + return nil, nil + } + tx, err := s.db.Begin() + if err != nil { + return nil, err + } + defer tx.Rollback() + out := make([]ArchiveResult, 0, len(ms)) + for _, m := range ms { + id, created, err := archiveOne(tx, listID, m) + if err != nil { + return nil, err + } + out = append(out, ArchiveResult{ID: id, Created: created}) + } + return out, tx.Commit() +} + +func archiveOne(q querier, listID int64, m Message) (id int64, created bool, err error) { if m.MessageID == "" { return 0, false, errors.New("message id is required") } - err = s.db.QueryRow(`SELECT id FROM archive WHERE list_id=? AND message_id=?`, listID, m.MessageID).Scan(&id) + err = q.QueryRow(`SELECT id FROM archive WHERE list_id=? AND message_id=?`, listID, m.MessageID).Scan(&id) if err == nil { return id, false, nil } @@ -606,12 +644,12 @@ func (s *Store) ArchiveMessage(listID int64, m Message) (id int64, created bool, } var thread int64 if m.InReplyTo != "" { - e := s.db.QueryRow(`SELECT thread_id FROM archive WHERE list_id=? AND message_id=?`, listID, m.InReplyTo).Scan(&thread) + e := q.QueryRow(`SELECT thread_id FROM archive WHERE list_id=? AND message_id=?`, listID, m.InReplyTo).Scan(&thread) if e != nil && !errors.Is(e, sql.ErrNoRows) { return 0, false, e } } - res, err := s.db.Exec(`INSERT INTO archive(list_id, thread_id, message_id, in_reply_to, subject, sender, date, body, size) + res, err := q.Exec(`INSERT INTO archive(list_id, thread_id, message_id, in_reply_to, subject, sender, date, body, size) VALUES(?,?,?,?,?,?,?,?,?)`, listID, thread, m.MessageID, m.InReplyTo, m.Subject, m.Sender, m.Date.Unix(), m.Body, len(m.Body)) if err != nil { @@ -619,13 +657,102 @@ func (s *Store) ArchiveMessage(listID int64, m Message) (id int64, created bool, } id, _ = res.LastInsertId() if thread == 0 { - if _, err := s.db.Exec(`UPDATE archive SET thread_id=id WHERE id=?`, id); err != nil { + if _, err := q.Exec(`UPDATE archive SET thread_id=id WHERE id=?`, id); err != nil { return 0, false, err } } return id, true, nil } +// Rethread recomputes the thread of every message of a list from the In-Reply-To +// links, and returns how many messages changed thread. Insertion order does not +// matter: a message imported before its parent ends up in the parent's thread. +// A cycle of replies becomes one thread whose id is the smallest id in the cycle. +func (s *Store) Rethread(listID int64) (int, error) { + rows, err := s.db.Query(`SELECT id, message_id, in_reply_to, thread_id FROM archive WHERE list_id=?`, listID) + if err != nil { + return 0, err + } + type row struct { + id, thread int64 + replyTo string + } + var all []row + byMsgID := map[string]int64{} + for rows.Next() { + var r row + var msgID string + if err := rows.Scan(&r.id, &msgID, &r.replyTo, &r.thread); err != nil { + rows.Close() + return 0, err + } + all = append(all, r) + byMsgID[msgID] = r.id + } + rows.Close() + if err := rows.Err(); err != nil { + return 0, err + } + + parent := map[int64]int64{} + for _, r := range all { + if p, ok := byMsgID[r.replyTo]; ok && r.replyTo != "" { + parent[r.id] = p + } + } + roots := map[int64]int64{} + root := func(start int64) int64 { + if r, ok := roots[start]; ok { + return r + } + path := []int64{start} + seen := map[int64]int{start: 0} + cur := start + var result int64 + for { + p, ok := parent[cur] + if !ok { + result = cur + break + } + if r, known := roots[p]; known { + result = r + break + } + if at, loop := seen[p]; loop { // a cycle: its smallest id is the root + result = path[at] + for _, id := range path[at:] { + result = min(result, id) + } + break + } + seen[p] = len(path) + path = append(path, p) + cur = p + } + for _, id := range path { + roots[id] = result + } + return result + } + + tx, err := s.db.Begin() + if err != nil { + return 0, err + } + defer tx.Rollback() + changed := 0 + for _, r := range all { + if want := root(r.id); want != r.thread { + if _, err := tx.Exec(`UPDATE archive SET thread_id=? WHERE id=?`, want, r.id); err != nil { + return 0, err + } + changed++ + } + } + return changed, tx.Commit() +} + // GetMessage returns an archived message with its body. func (s *Store) GetMessage(id int64) (Message, error) { return scanMessage(s.db.QueryRow(`SELECT `+msgCols+`, body FROM archive WHERE id=?`, id), true)