From ba6c37bc21786986c62454551008dc668dde3d56 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Mon, 9 Nov 2026 17:13:56 +0100 Subject: [PATCH 1/7] feat(mbox,store): mbox reader, batch archiving and thread rebuilding - Add internal/mbox: streaming reader with strict From_ detection, mboxrd unescaping, size cap and envelope dates - Add Store.ArchiveBatch: many messages in one transaction - Add Store.Rethread: rebuild threads from In-Reply-To whatever the insertion order, tolerating cycles --- internal/mbox/mbox.go | 147 +++++++++++++++++++++++++++++ internal/mbox/mbox_test.go | 165 +++++++++++++++++++++++++++++++++ internal/store/archive_test.go | 156 +++++++++++++++++++++++++++++++ internal/store/store.go | 135 ++++++++++++++++++++++++++- 4 files changed, 599 insertions(+), 4 deletions(-) create mode 100644 internal/mbox/mbox.go create mode 100644 internal/mbox/mbox_test.go diff --git a/internal/mbox/mbox.go b/internal/mbox/mbox.go new file mode 100644 index 0000000..a6aa4b6 --- /dev/null +++ b/internal/mbox/mbox.go @@ -0,0 +1,147 @@ +// Package mbox reads mbox archives, such as the ones Mailman exports. +// +// A message starts at a line like "From alice@example.com Tue Jan 2 15:04:05 2024". +// A line that merely begins with "From " in prose does not split a message. Lines +// escaped as ">From " are restored (mboxrd); the blank line before a separator +// belongs to the separator, not to the message. +package mbox + +import ( + "bufio" + "bytes" + "errors" + "io" + "regexp" + "strings" + "time" +) + +// ErrTooLarge is returned with a message that exceeded the size limit and was skipped. +var ErrTooLarge = errors.New("message too large") + +var ( + separator = regexp.MustCompile(`^From \S+\s+(Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2}\s+\d{1,2}:\d{2}`) + escaped = regexp.MustCompile(`^>+From `) +) + +// Message is one message of the mbox. +type Message struct { + Index int // 1-based position in the file + Envelope string // the "From " line + Data []byte // the RFC 5322 message; nil when Err is set + Err error +} + +// Date returns the date of the "From " line, when it has a parsable one. +func (m Message) Date() (time.Time, bool) { + f := strings.Fields(m.Envelope) + if len(f) < 3 || f[0] != "From" { + return time.Time{}, false + } + rest := strings.Join(f[2:], " ") + for _, layout := range []string{ + "Mon Jan 2 15:04:05 2006", + "Mon Jan 2 15:04:05 -0700 2006", + "Mon Jan 2 15:04:05 MST 2006", + "Mon Jan 2 15:04 2006", + } { + if t, err := time.Parse(layout, rest); err == nil { + return t, true + } + } + return time.Time{}, false +} + +// Reader returns the messages of an mbox one at a time. +type Reader struct { + br *bufio.Reader + max int + next string // envelope of the message that comes next, once seen + index int + eof bool +} + +// NewReader reads from r. Messages larger than maxSize bytes (0 means no limit) +// are skipped and reported with ErrTooLarge. +func NewReader(r io.Reader, maxSize int) *Reader { + return &Reader{br: bufio.NewReaderSize(r, 64*1024), max: maxSize} +} + +// readLine returns the next line including its newline. At the end of the input +// it returns the last partial line (if any) with io.EOF on the following call. +func (r *Reader) readLine() ([]byte, error) { + line, err := r.br.ReadBytes('\n') + if err == io.EOF && len(line) > 0 { + return line, nil + } + return line, err +} + +// Next returns the next message, or io.EOF when there are no more. +func (r *Reader) Next() (Message, error) { + if r.eof { + return Message{}, io.EOF + } + // Skip anything before the first separator. + for r.next == "" { + line, err := r.readLine() + if err != nil { + r.eof = true + if err == io.EOF { + return Message{}, io.EOF + } + return Message{}, err + } + if s := strings.TrimRight(string(line), "\r\n"); separator.MatchString(s) { + r.next = s + } + } + r.index++ + msg := Message{Index: r.index, Envelope: r.next} + r.next = "" + + var buf bytes.Buffer + oversized := false + for { + line, err := r.readLine() + if err != nil { + r.eof = true + if err != io.EOF { + return Message{}, err + } + break + } + if s := strings.TrimRight(string(line), "\r\n"); separator.MatchString(s) { + r.next = s + break + } + if oversized { + continue + } + if escaped.Match(line) { + line = line[1:] + } + buf.Write(line) + if r.max > 0 && buf.Len() > r.max { + oversized = true + buf.Reset() + } + } + if oversized { + msg.Err = ErrTooLarge + return msg, ErrTooLarge + } + msg.Data = trimSeparatorLine(buf.Bytes()) + return msg, nil +} + +// trimSeparatorLine drops the one blank line that precedes a separator. +func trimSeparatorLine(b []byte) []byte { + switch { + case bytes.HasSuffix(b, []byte("\r\n\r\n")): + return b[:len(b)-2] + case bytes.HasSuffix(b, []byte("\n\n")): + return b[:len(b)-1] + } + return b +} diff --git a/internal/mbox/mbox_test.go b/internal/mbox/mbox_test.go new file mode 100644 index 0000000..c508f43 --- /dev/null +++ b/internal/mbox/mbox_test.go @@ -0,0 +1,165 @@ +package mbox + +import ( + "errors" + "io" + "strings" + "testing" + "time" +) + +func readAll(t *testing.T, input string, maxSize int) []Message { + t.Helper() + r := NewReader(strings.NewReader(input), maxSize) + var out []Message + for { + m, err := r.Next() + if err == io.EOF { + return out + } + if err != nil && !errors.Is(err, ErrTooLarge) { + t.Fatalf("next: %v", err) + } + m.Err = err + out = append(out, m) + } +} + +const two = "From alice@example.com Tue Jan 2 15:04:05 2024\n" + + "From: alice@example.com\nSubject: one\n\nfirst body\n\n" + + "From bob@example.com Wed Jan 3 09:00:00 2024\n" + + "From: bob@example.com\nSubject: two\n\nsecond body\n" + +func TestSplitsMessages(t *testing.T) { + ms := readAll(t, two, 0) + if len(ms) != 2 { + t.Fatalf("got %d messages", len(ms)) + } + if string(ms[0].Data) != "From: alice@example.com\nSubject: one\n\nfirst body\n" { + t.Fatalf("first = %q", ms[0].Data) + } + if string(ms[1].Data) != "From: bob@example.com\nSubject: two\n\nsecond body\n" { + t.Fatalf("second = %q", ms[1].Data) + } + if ms[0].Envelope != "From alice@example.com Tue Jan 2 15:04:05 2024" { + t.Fatalf("envelope = %q", ms[0].Envelope) + } +} + +func TestEnvelopeDate(t *testing.T) { + ms := readAll(t, two, 0) + want := time.Date(2024, 1, 2, 15, 4, 5, 0, time.UTC) + if got, ok := ms[0].Date(); !ok || !got.Equal(want) { + t.Fatalf("date = %v %v", got, ok) + } + for _, env := range []string{ + "From x@y.z Tue Jan 2 15:04:05 +0200 2024", + "From x@y.z Tue Jan 2 15:04:05 2024", + "From x@y.z Tue Jan 2 15:04 2024", + "From MAILER-DAEMON Tue Jan 2 15:04:05 2024", + } { + if _, ok := (Message{Envelope: env}).Date(); !ok { + t.Errorf("%q must parse", env) + } + } + for _, env := range []string{"", "From x@y.z", "From x@y.z not a date"} { + if _, ok := (Message{Envelope: env}).Date(); ok { + t.Errorf("%q must not parse", env) + } + } +} + +func TestProseLineStartingWithFromDoesNotSplit(t *testing.T) { + in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nFrom now on we meet on Fridays.\nFrom: not a separator either\n" + ms := readAll(t, in, 0) + if len(ms) != 1 || !strings.Contains(string(ms[0].Data), "From now on we meet on Fridays.") { + t.Fatalf("messages: %d %q", len(ms), ms[0].Data) + } +} + +func TestMboxrdUnescaping(t *testing.T) { + in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n>From the log:\n>>From a quoted line\n> From not escaped\nplain\n" + ms := readAll(t, in, 0) + body := string(ms[0].Data) + if !strings.Contains(body, "\nFrom the log:\n") || !strings.Contains(body, "\n>From a quoted line\n") || !strings.Contains(body, "\n> From not escaped\n") { + t.Fatalf("body = %q", body) + } +} + +func TestLeadingJunkAndEmptyInput(t *testing.T) { + if got := readAll(t, "", 0); len(got) != 0 { + t.Fatalf("empty input: %d", len(got)) + } + if got := readAll(t, "\n\nsome preamble\nFrom a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nbody\n", 0); len(got) != 1 { + t.Fatalf("preamble must be ignored: %d", len(got)) + } + if got := readAll(t, "just text, no separator at all\n", 0); len(got) != 0 { + t.Fatalf("no messages expected: %d", len(got)) + } +} + +func TestCRLFAndMissingFinalNewline(t *testing.T) { + in := "From a@x.com Tue Jan 2 15:04:05 2024\r\nSubject: s\r\n\r\nbody without newline" + ms := readAll(t, in, 0) + if len(ms) != 1 || !strings.HasSuffix(string(ms[0].Data), "body without newline") || !strings.Contains(string(ms[0].Data), "Subject: s\r\n") { + t.Fatalf("%q", ms[0].Data) + } + crlf := "From a@x.com Tue Jan 2 15:04:05 2024\r\nSubject: one\r\n\r\nb\r\n\r\nFrom b@x.com Tue Jan 2 15:04:05 2024\r\nSubject: two\r\n\r\nb2\r\n" + if got := readAll(t, crlf, 0); len(got) != 2 { + t.Fatalf("CRLF separators: %d", len(got)) + } +} + +func TestVeryLongLinesAreFine(t *testing.T) { + long := strings.Repeat("x", 300_000) + ms := readAll(t, "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n"+long+"\n", 0) + if len(ms) != 1 || !strings.Contains(string(ms[0].Data), long) { + t.Fatal("a long line must survive") + } +} + +func TestOversizedMessagesAreReportedAndSkipped(t *testing.T) { + big := strings.Repeat("y\n", 5000) + in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: ok1\n\nsmall\n" + + "From b@x.com Tue Jan 2 15:04:05 2024\nSubject: big\n\n" + big + + "From c@x.com Tue Jan 2 15:04:05 2024\nSubject: ok2\n\nsmall\n" + ms := readAll(t, in, 1000) + if len(ms) != 3 { + t.Fatalf("got %d", len(ms)) + } + if ms[0].Err != nil || ms[2].Err != nil { + t.Fatalf("small messages must pass: %v %v", ms[0].Err, ms[2].Err) + } + if !errors.Is(ms[1].Err, ErrTooLarge) || ms[1].Data != nil { + t.Fatalf("big message: %v %d bytes", ms[1].Err, len(ms[1].Data)) + } + if ms[1].Envelope != "From b@x.com Tue Jan 2 15:04:05 2024" || !strings.Contains(string(ms[2].Data), "ok2") { + t.Fatalf("the reader must resynchronise after a big message: %+v", ms[2]) + } +} + +func TestIndexIsOneBased(t *testing.T) { + ms := readAll(t, two, 0) + if ms[0].Index != 1 || ms[1].Index != 2 { + t.Fatalf("indexes: %d %d", ms[0].Index, ms[1].Index) + } +} + +func TestReadErrorsArePropagated(t *testing.T) { + r := NewReader(io.MultiReader(strings.NewReader("From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nbody\n"), errReader{}), 0) + var last error + for i := 0; i < 5; i++ { + _, err := r.Next() + if err != nil { + last = err + break + } + } + if last == nil || last == io.EOF { + t.Fatalf("want the read error, got %v", last) + } +} + +type errReader struct{} + +func (errReader) Read([]byte) (int, error) { return 0, errors.New("disk on fire") } diff --git a/internal/store/archive_test.go b/internal/store/archive_test.go index 7fe6b01..1ee9133 100644 --- a/internal/store/archive_test.go +++ b/internal/store/archive_test.go @@ -227,3 +227,159 @@ func TestLongBodyIsStoredAsIs(t *testing.T) { t.Fatalf("body len %d", len(got.Body)) } } + +func TestArchiveBatchInsertsDeduplicatesAndThreads(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.ArchiveMessage(l.ID, msg("", "Already here", "x")) + + reply := msg("", "Re: Topic", "reply") + reply.InReplyTo = "" + res, err := s.ArchiveBatch(l.ID, []Message{ + msg("", "Topic", "root"), + reply, + msg("", "Already here", "x"), // duplicate of an existing message + msg("", "Topic", "root"), // duplicate inside the batch + }) + if err != nil { + t.Fatal(err) + } + if len(res) != 4 || !res[0].Created || !res[1].Created || res[2].Created || res[3].Created { + t.Fatalf("results: %+v", res) + } + if res[3].ID != res[0].ID { + t.Fatalf("a duplicate reports the id of the stored message: %+v", res) + } + r, _ := s.GetMessage(res[1].ID) + if r.ThreadID != res[0].ID { + t.Fatalf("threading inside a batch: thread %d, root %d", r.ThreadID, res[0].ID) + } + if _, total, _ := s.ListMessages(l.ID, 10, 0); total != 3 { + t.Fatalf("total = %d", total) + } + if got, _, _ := s.SearchMessages(l.ID, "reply", 10, 0); len(got) != 1 { + t.Fatal("batch inserts must be searchable") + } +} + +func TestArchiveBatchIsAtomic(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + _, err := s.ArchiveBatch(l.ID, []Message{msg("", "A", "a"), {Subject: "no id"}}) + if err == nil { + t.Fatal("a message without an id must fail the batch") + } + if _, total, _ := s.ListMessages(l.ID, 10, 0); total != 0 { + t.Fatalf("nothing may be stored after a failed batch, got %d", total) + } + if res, err := s.ArchiveBatch(l.ID, nil); err != nil || len(res) != 0 { + t.Fatalf("empty batch: %v %v", res, err) + } +} + +func threadOf(t *testing.T, s *Store, id int64) int64 { + t.Helper() + m, err := s.GetMessage(id) + if err != nil { + t.Fatal(err) + } + return m.ThreadID +} + +func TestRethreadFixesMessagesImportedOutOfOrder(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + // Children first, root last: at insertion time none of the parents exists. + c2 := msg("", "Re: Re: T", "c2") + c2.InReplyTo = "" + c1 := msg("", "Re: T", "c1") + c1.InReplyTo = "" + id2, _, _ := s.ArchiveMessage(l.ID, c2) + id1, _, _ := s.ArchiveMessage(l.ID, c1) + root, _, _ := s.ArchiveMessage(l.ID, msg("", "T", "root")) + other, _, _ := s.ArchiveMessage(l.ID, msg("", "Other", "o")) + if threadOf(t, s, id1) == root { + t.Fatal("precondition: the thread is broken before rethreading") + } + + changed, err := s.Rethread(l.ID) + if err != nil { + t.Fatal(err) + } + if changed != 2 { + t.Fatalf("changed = %d, want the 2 broken messages", changed) + } + for _, id := range []int64{id1, id2, root} { + if got := threadOf(t, s, id); got != root { + t.Fatalf("message %d thread = %d, want %d", id, got, root) + } + } + if threadOf(t, s, other) != other { + t.Fatal("an unrelated message keeps its own thread") + } + if again, _ := s.Rethread(l.ID); again != 0 { + t.Fatalf("rethreading twice changes nothing, got %d", again) + } + if th, _ := s.ThreadMessages(l.ID, root); len(th) != 3 { + t.Fatalf("thread has %d messages", len(th)) + } +} + +func TestRethreadSurvivesCyclesAndSelfReplies(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + a := msg("", "A", "a") + a.InReplyTo = "" + b := msg("", "B", "b") + b.InReplyTo = "" + self := msg("", "Self", "s") + self.InReplyTo = "" + ia, _, _ := s.ArchiveMessage(l.ID, a) + ib, _, _ := s.ArchiveMessage(l.ID, b) + is, _, _ := s.ArchiveMessage(l.ID, self) + + done := make(chan error, 1) + go func() { _, err := s.Rethread(l.ID); done <- err }() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(3 * time.Second): + t.Fatal("Rethread looped forever on a cycle") + } + if threadOf(t, s, ia) != threadOf(t, s, ib) { + t.Fatal("the two messages of a cycle share a thread") + } + if got := threadOf(t, s, ia); got != ia { + t.Fatalf("the cycle's thread is its smallest id (%d), got %d", ia, got) + } + if threadOf(t, s, is) != is { + t.Fatal("a message that replies to itself is its own thread") + } +} + +func TestRethreadOnlyTouchesTheGivenList(t *testing.T) { + s := newTestStore(t) + a, _ := s.CreateList("a", "example.org", "A") + b, _ := s.CreateList("b", "example.org", "B") + child := msg("", "Re", "c") + child.InReplyTo = "" + cb, _, _ := s.ArchiveMessage(b.ID, child) + s.ArchiveMessage(b.ID, msg("", "P", "p")) + // A parent with the same Message-ID in another list must not be used. + s.ArchiveMessage(a.ID, msg("", "P in a", "p")) + + if n, _ := s.Rethread(a.ID); n != 0 { + t.Fatalf("list a has nothing to fix, changed %d", n) + } + if threadOf(t, s, cb) != cb { + t.Fatal("list b must be untouched by rethreading list a") + } + if n, _ := s.Rethread(b.ID); n != 1 { + t.Fatalf("list b: changed %d", n) + } + if threadOf(t, s, cb) == cb { + t.Fatal("list b's child must now follow its parent") + } +} diff --git a/internal/store/store.go b/internal/store/store.go index ec595f1..b14b2b0 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -590,14 +590,52 @@ func scanMessage(row interface{ Scan(...any) error }, withBody bool) (Message, e return m, err } +// querier is what archiveOne needs from a *sql.DB or a *sql.Tx. +type querier interface { + Exec(query string, args ...any) (sql.Result, error) + QueryRow(query string, args ...any) *sql.Row +} + +// ArchiveResult is the outcome for one message of ArchiveBatch. +type ArchiveResult struct { + ID int64 + Created bool +} + // ArchiveMessage stores a post. A message already archived in the same list // (same Message-ID) is not stored again: created is false and the existing id // is returned, so MTA retries are harmless. func (s *Store) ArchiveMessage(listID int64, m Message) (id int64, created bool, err error) { + return archiveOne(s.db, listID, m) +} + +// ArchiveBatch stores many messages in one transaction: all or none. It is much +// faster than ArchiveMessage in a loop, because SQLite syncs once per commit. +func (s *Store) ArchiveBatch(listID int64, ms []Message) ([]ArchiveResult, error) { + if len(ms) == 0 { + return nil, nil + } + tx, err := s.db.Begin() + if err != nil { + return nil, err + } + defer tx.Rollback() + out := make([]ArchiveResult, 0, len(ms)) + for _, m := range ms { + id, created, err := archiveOne(tx, listID, m) + if err != nil { + return nil, err + } + out = append(out, ArchiveResult{ID: id, Created: created}) + } + return out, tx.Commit() +} + +func archiveOne(q querier, listID int64, m Message) (id int64, created bool, err error) { if m.MessageID == "" { return 0, false, errors.New("message id is required") } - err = s.db.QueryRow(`SELECT id FROM archive WHERE list_id=? AND message_id=?`, listID, m.MessageID).Scan(&id) + err = q.QueryRow(`SELECT id FROM archive WHERE list_id=? AND message_id=?`, listID, m.MessageID).Scan(&id) if err == nil { return id, false, nil } @@ -606,12 +644,12 @@ func (s *Store) ArchiveMessage(listID int64, m Message) (id int64, created bool, } var thread int64 if m.InReplyTo != "" { - e := s.db.QueryRow(`SELECT thread_id FROM archive WHERE list_id=? AND message_id=?`, listID, m.InReplyTo).Scan(&thread) + e := q.QueryRow(`SELECT thread_id FROM archive WHERE list_id=? AND message_id=?`, listID, m.InReplyTo).Scan(&thread) if e != nil && !errors.Is(e, sql.ErrNoRows) { return 0, false, e } } - res, err := s.db.Exec(`INSERT INTO archive(list_id, thread_id, message_id, in_reply_to, subject, sender, date, body, size) + res, err := q.Exec(`INSERT INTO archive(list_id, thread_id, message_id, in_reply_to, subject, sender, date, body, size) VALUES(?,?,?,?,?,?,?,?,?)`, listID, thread, m.MessageID, m.InReplyTo, m.Subject, m.Sender, m.Date.Unix(), m.Body, len(m.Body)) if err != nil { @@ -619,13 +657,102 @@ func (s *Store) ArchiveMessage(listID int64, m Message) (id int64, created bool, } id, _ = res.LastInsertId() if thread == 0 { - if _, err := s.db.Exec(`UPDATE archive SET thread_id=id WHERE id=?`, id); err != nil { + if _, err := q.Exec(`UPDATE archive SET thread_id=id WHERE id=?`, id); err != nil { return 0, false, err } } return id, true, nil } +// Rethread recomputes the thread of every message of a list from the In-Reply-To +// links, and returns how many messages changed thread. Insertion order does not +// matter: a message imported before its parent ends up in the parent's thread. +// A cycle of replies becomes one thread whose id is the smallest id in the cycle. +func (s *Store) Rethread(listID int64) (int, error) { + rows, err := s.db.Query(`SELECT id, message_id, in_reply_to, thread_id FROM archive WHERE list_id=?`, listID) + if err != nil { + return 0, err + } + type row struct { + id, thread int64 + replyTo string + } + var all []row + byMsgID := map[string]int64{} + for rows.Next() { + var r row + var msgID string + if err := rows.Scan(&r.id, &msgID, &r.replyTo, &r.thread); err != nil { + rows.Close() + return 0, err + } + all = append(all, r) + byMsgID[msgID] = r.id + } + rows.Close() + if err := rows.Err(); err != nil { + return 0, err + } + + parent := map[int64]int64{} + for _, r := range all { + if p, ok := byMsgID[r.replyTo]; ok && r.replyTo != "" { + parent[r.id] = p + } + } + roots := map[int64]int64{} + root := func(start int64) int64 { + if r, ok := roots[start]; ok { + return r + } + path := []int64{start} + seen := map[int64]int{start: 0} + cur := start + var result int64 + for { + p, ok := parent[cur] + if !ok { + result = cur + break + } + if r, known := roots[p]; known { + result = r + break + } + if at, loop := seen[p]; loop { // a cycle: its smallest id is the root + result = path[at] + for _, id := range path[at:] { + result = min(result, id) + } + break + } + seen[p] = len(path) + path = append(path, p) + cur = p + } + for _, id := range path { + roots[id] = result + } + return result + } + + tx, err := s.db.Begin() + if err != nil { + return 0, err + } + defer tx.Rollback() + changed := 0 + for _, r := range all { + if want := root(r.id); want != r.thread { + if _, err := tx.Exec(`UPDATE archive SET thread_id=? WHERE id=?`, want, r.id); err != nil { + return 0, err + } + changed++ + } + } + return changed, tx.Commit() +} + // GetMessage returns an archived message with its body. func (s *Store) GetMessage(id int64) (Message, error) { return scanMessage(s.db.QueryRow(`SELECT `+msgCols+`, body FROM archive WHERE id=?`, id), true) From 45a61ef29459464ad2ad744b7aa587c3da186e90 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Mon, 9 Nov 2026 17:14:46 +0100 Subject: [PATCH 2/7] feat(archive): import messages from an mbox - Add ImportMbox: batches of 200, idempotent by Message-ID, threads rebuilt at the end - Date fallbacks: Date header, then the From line, then the import time - Report failed and oversized messages without stopping, keep progress on a read error - Import regardless of the list's archive policy --- internal/archive/import.go | 106 +++++++++++++ internal/archive/import_test.go | 254 ++++++++++++++++++++++++++++++++ 2 files changed, 360 insertions(+) create mode 100644 internal/archive/import.go create mode 100644 internal/archive/import_test.go diff --git a/internal/archive/import.go b/internal/archive/import.go new file mode 100644 index 0000000..ce2a4c5 --- /dev/null +++ b/internal/archive/import.go @@ -0,0 +1,106 @@ +package archive + +import ( + "errors" + "fmt" + "io" + "time" + + "codeberg.org/squidd/parsimony/internal/mbox" + "codeberg.org/squidd/parsimony/internal/store" +) + +const ( + // MaxImportMessage is the largest message accepted from an mbox, in bytes. + MaxImportMessage = 25 << 20 + // MaxReportedErrors caps the error messages kept in a summary; Failed still counts every one. + MaxReportedErrors = 20 + + importBatch = 200 // messages per transaction +) + +// ImportSummary reports what an import did. +type ImportSummary struct { + Imported int `json:"imported"` // new messages stored + Duplicates int `json:"duplicates"` // already in the archive (same Message-ID) + Failed int `json:"failed"` // unreadable or too large + Errors []string `json:"errors"` // the first MaxReportedErrors failures +} + +// ImportMbox archives every message of an mbox into a list. It is idempotent +// (a message is identified by its Message-ID) and works whatever the list's +// archive policy is: the policy only governs live mail. Threads are rebuilt at +// the end, so the order of the messages in the file does not matter. now is the +// date of last resort for a message with neither a Date header nor a dated +// "From " line. On a read error the messages read so far are kept and returned +// in the summary together with the error. +func ImportMbox(st *store.Store, listID int64, r io.Reader, now time.Time) (ImportSummary, error) { + sum := ImportSummary{Errors: []string{}} + var batch []store.Message + + fail := func(index int, err error) { + sum.Failed++ + if len(sum.Errors) < MaxReportedErrors { + sum.Errors = append(sum.Errors, fmt.Sprintf("message %d: %v", index, err)) + } + } + flush := func() error { + results, err := st.ArchiveBatch(listID, batch) + if err != nil { + return err + } + for _, res := range results { + if res.Created { + sum.Imported++ + } else { + sum.Duplicates++ + } + } + batch = batch[:0] + return nil + } + finish := func(readErr error) (ImportSummary, error) { + if err := flush(); err != nil { + return sum, err + } + if sum.Imported > 0 { + if _, err := st.Rethread(listID); err != nil { + return sum, err + } + } + return sum, readErr + } + + rd := mbox.NewReader(r, MaxImportMessage) + for { + m, err := rd.Next() + if err == io.EOF { + return finish(nil) + } + if errors.Is(err, mbox.ErrTooLarge) { + fail(m.Index, fmt.Errorf("too large (limit %d MiB)", MaxImportMessage>>20)) + continue + } + if err != nil { + return finish(fmt.Errorf("read mbox: %w", err)) + } + fallback := now + if d, ok := m.Date(); ok { + fallback = d + } + en, err := Parse(m.Data, fallback) + if err != nil { + fail(m.Index, err) + continue + } + batch = append(batch, store.Message{ + MessageID: en.MessageID, InReplyTo: en.InReplyTo, Subject: en.Subject, + Sender: en.Sender, Date: en.Date, Body: en.Body, + }) + if len(batch) >= importBatch { + if err := flush(); err != nil { + return sum, err + } + } + } +} diff --git a/internal/archive/import_test.go b/internal/archive/import_test.go new file mode 100644 index 0000000..6526227 --- /dev/null +++ b/internal/archive/import_test.go @@ -0,0 +1,254 @@ +package archive + +import ( + "errors" + "fmt" + "strings" + "testing" + "time" + + "codeberg.org/squidd/parsimony/internal/store" +) + +var importNow = time.Date(2031, 5, 6, 7, 8, 9, 0, time.UTC) + +func newList(t *testing.T) (*store.Store, store.List) { + t.Helper() + s, err := store.Open(":memory:") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + l, _ := s.CreateList("dev", "example.org", "Dev") + return s, l +} + +func mboxMsg(from, subject, id, replyTo, date, body string) string { + addr := from + if i := strings.Index(from, "<"); i >= 0 { + addr = strings.Trim(from[i:], "<>") + } + h := "From " + addr + " Tue Jan 2 15:04:05 2024\n" + h += "From: " + from + "\nSubject: " + subject + "\n" + if id != "" { + h += "Message-ID: " + id + "\n" + } + if replyTo != "" { + h += "In-Reply-To: " + replyTo + "\n" + } + if date != "" { + h += "Date: " + date + "\n" + } + return h + "\n" + body + "\n\n" +} + +func TestImportMbox(t *testing.T) { + s, l := newList(t) + mbox := mboxMsg("Alice ", "Kickoff", "<1@x>", "", "Mon, 01 Jan 2024 10:00:00 +0000", "let's start") + + mboxMsg("Bob ", "Re: Kickoff", "<2@x>", "<1@x>", "Mon, 01 Jan 2024 11:00:00 +0000", "agreed") + + mboxMsg("Carol ", "Lunch", "<3@x>", "", "Tue, 02 Jan 2024 12:00:00 +0000", "pizza?") + sum, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow) + if err != nil { + t.Fatal(err) + } + if sum.Imported != 3 || sum.Duplicates != 0 || sum.Failed != 0 || len(sum.Errors) != 0 { + t.Fatalf("summary: %+v", sum) + } + ms, total, _ := s.ListMessages(l.ID, 10, 0) + if total != 3 || ms[0].Subject != "Lunch" { + t.Fatalf("messages: %+v", ms) + } + var reply store.Message + for _, m := range ms { + if m.Subject == "Re: Kickoff" { + reply, _ = s.GetMessage(m.ID) + } + } + if reply.Sender != "Bob" || reply.Body != "agreed\n" || !reply.Date.Equal(time.Date(2024, 1, 1, 11, 0, 0, 0, time.UTC)) { + t.Fatalf("reply: %+v", reply) + } + root, _ := s.GetMessage(reply.ThreadID) + if root.Subject != "Kickoff" { + t.Fatalf("the reply must be threaded under its parent, got %q", root.Subject) + } +} + +func TestImportIsIdempotent(t *testing.T) { + s, l := newList(t) + mbox := mboxMsg("A ", "One", "<1@x>", "", "", "b1") + mboxMsg("B ", "Two", "", "", "", "b2") + if sum, _ := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow); sum.Imported != 2 { + t.Fatalf("first: %+v", sum) + } + sum, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow) + if err != nil || sum.Imported != 0 || sum.Duplicates != 2 || sum.Failed != 0 { + t.Fatalf("second import must add nothing, even for a message without a Message-ID: %+v %v", sum, err) + } + if _, total, _ := s.ListMessages(l.ID, 10, 0); total != 2 { + t.Fatalf("total = %d", total) + } +} + +func TestImportRebuildsThreadsWhateverTheOrder(t *testing.T) { + s, l := newList(t) + mbox := mboxMsg("C ", "Re: Re: T", "", "", "", "c") + + mboxMsg("B ", "Re: T", "", "", "", "b") + + mboxMsg("A ", "T", "", "", "", "a") + if _, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow); err != nil { + t.Fatal(err) + } + ms, _, _ := s.ListMessages(l.ID, 10, 0) + threads := map[int64]int{} + for _, m := range ms { + threads[m.ThreadID]++ + } + if len(threads) != 1 { + t.Fatalf("one thread expected, got %v", threads) + } +} + +func TestImportReportsBadMessagesAndKeepsGoing(t *testing.T) { + s, l := newList(t) + mbox := mboxMsg("A ", "Good one", "<1@x>", "", "", "ok") + + "From x@y.z Tue Jan 2 15:04:05 2024\n\n\nthis message has no headers at all\n\n" + + mboxMsg("B ", "Good two", "<2@x>", "", "", "ok") + sum, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow) + if err != nil { + t.Fatal(err) + } + if sum.Imported != 2 || sum.Failed != 1 || len(sum.Errors) != 1 || !strings.Contains(sum.Errors[0], "message 2") { + t.Fatalf("summary: %+v", sum) + } +} + +func TestImportSkipsOversizedMessages(t *testing.T) { + s, l := newList(t) + big := strings.Repeat("0123456789", MaxImportMessage/10+10) + mbox := mboxMsg("A ", "Small", "<1@x>", "", "", "ok") + mboxMsg("B ", "Huge", "<2@x>", "", "", big) + + mboxMsg("C ", "Small too", "<3@x>", "", "", "ok") + sum, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow) + if err != nil { + t.Fatal(err) + } + if sum.Imported != 2 || sum.Failed != 1 || !strings.Contains(sum.Errors[0], "message 2") || !strings.Contains(sum.Errors[0], "too large") { + t.Fatalf("summary: %+v", sum) + } +} + +func TestImportDateFallbacks(t *testing.T) { + s, l := newList(t) + mbox := mboxMsg("A ", "has Date", "<1@x>", "", "Mon, 01 Jan 2024 10:00:00 +0000", "b") + + mboxMsg("B ", "from envelope", "<2@x>", "", "", "b") + + "From c@x.com Fri Feb 23 08:30 2024\nFrom: C \nSubject: nothing\nMessage-ID: <3@x>\n\nb\n\n" + if _, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow); err != nil { + t.Fatal(err) + } + ms, _, _ := s.ListMessages(l.ID, 10, 0) + dates := map[string]time.Time{} + for _, m := range ms { + dates[m.Subject] = m.Date + } + if !dates["has Date"].Equal(time.Date(2024, 1, 1, 10, 0, 0, 0, time.UTC)) { + t.Errorf("Date header wins: %v", dates["has Date"]) + } + if !dates["from envelope"].Equal(time.Date(2024, 1, 2, 15, 4, 5, 0, time.UTC)) { + t.Errorf("the From line is the fallback: %v", dates["from envelope"]) + } + if !dates["nothing"].Equal(time.Date(2024, 2, 23, 8, 30, 0, 0, time.UTC)) { + t.Errorf("minute-precision envelope date: %v", dates["nothing"]) + } +} + +func TestImportDateFallsBackToNowAsALastResort(t *testing.T) { + s, l := newList(t) + // A valid separator whose date has trailing junk, so it cannot be parsed, and no Date header. + mbox := "From a@x.com Tue Jan 2 15:04:05 2024 extra junk after\nFrom: A \nSubject: undated\nMessage-ID: <1@x>\n\nb\n" + sum, _ := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow) + if sum.Imported != 1 { + t.Fatalf("%+v", sum) + } + ms, _, _ := s.ListMessages(l.ID, 10, 0) + if !ms[0].Date.Equal(importNow) { + t.Fatalf("date = %v, want the import time", ms[0].Date) + } +} + +func TestImportCapsTheErrorListButCountsEverything(t *testing.T) { + s, l := newList(t) + var b strings.Builder + for i := 0; i < 30; i++ { + b.WriteString("From x@y.z Tue Jan 2 15:04:05 2024\n\nno headers here\n\n") + } + sum, err := ImportMbox(s, l.ID, strings.NewReader(b.String()), importNow) + if err != nil { + t.Fatal(err) + } + if sum.Failed != 30 || len(sum.Errors) != MaxReportedErrors { + t.Fatalf("failed=%d errors=%d", sum.Failed, len(sum.Errors)) + } +} + +func TestImportWorksWhateverTheArchivePolicy(t *testing.T) { + s, l := newList(t) + s.SetPolicies(l.ID, store.PostMembers, store.JoinOpen, store.ArchiveOff) + sum, _ := ImportMbox(s, l.ID, strings.NewReader(mboxMsg("A ", "S", "<1@x>", "", "", "b")), importNow) + if sum.Imported != 1 { + t.Fatalf("an explicit import is not governed by the live-mail policy: %+v", sum) + } +} + +func TestImportLargeFileCrossesBatchBoundaries(t *testing.T) { + s, l := newList(t) + var b strings.Builder + n := importBatch*2 + 50 + for i := 0; i < n; i++ { + parent := "" + if i > 0 && i%7 != 0 { + parent = fmt.Sprintf("", i-1) + } + b.WriteString(mboxMsg("A ", fmt.Sprintf("Subject %d", i), fmt.Sprintf("", i), parent, "", "body")) + } + sum, err := ImportMbox(s, l.ID, strings.NewReader(b.String()), importNow) + if err != nil || sum.Imported != n || sum.Failed != 0 { + t.Fatalf("summary: %+v %v", sum, err) + } + if _, total, _ := s.ListMessages(l.ID, 1, 0); total != n { + t.Fatalf("stored %d of %d", total, n) + } +} + +func TestImportDoesNotTouchOtherLists(t *testing.T) { + s, l := newList(t) + other, _ := s.CreateList("ops", "example.org", "Ops") + ImportMbox(s, l.ID, strings.NewReader(mboxMsg("A ", "S", "<1@x>", "", "", "b")), importNow) + if _, total, _ := s.ListMessages(other.ID, 10, 0); total != 0 { + t.Fatalf("other list has %d messages", total) + } +} + +type brokenReader struct { + data string + done bool +} + +func (b *brokenReader) Read(p []byte) (int, error) { + if !b.done { + b.done = true + return copy(p, b.data), nil + } + return 0, errors.New("connection reset") +} + +func TestImportReturnsTheProgressMadeBeforeAReadError(t *testing.T) { + s, l := newList(t) + data := mboxMsg("A ", "First", "<1@x>", "", "", "b") + mboxMsg("B ", "Second", "<2@x>", "", "", "b") + sum, err := ImportMbox(s, l.ID, &brokenReader{data: data}, importNow) + if err == nil || !strings.Contains(err.Error(), "connection reset") { + t.Fatalf("want the read error, got %v", err) + } + if sum.Imported != 1 { + t.Fatalf("the messages read before the failure are kept: %+v", sum) + } + if _, total, _ := s.ListMessages(l.ID, 10, 0); total != 1 { + t.Fatalf("stored %d", total) + } +} From cd596e181081e79886c1b25e1b0f7cc9d941fde3 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Mon, 9 Nov 2026 17:15:05 +0100 Subject: [PATCH 3/7] feat(api): import an mbox into a list's archive - Add POST /api/v1/lists/{id}/archive/import, streaming the body - Return a summary of imported, duplicate and failed messages; failures do not fail the request - Import regardless of the archive policy --- internal/api/api.go | 22 ++++++++++++ internal/api/api_test.go | 74 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) diff --git a/internal/api/api.go b/internal/api/api.go index 04fad59..e9283ef 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -7,6 +7,7 @@ import ( "crypto/subtle" "encoding/json" "errors" + "fmt" "mime" "net/http" "net/mail" @@ -14,6 +15,7 @@ import ( "strings" "time" + "codeberg.org/squidd/parsimony/internal/archive" "codeberg.org/squidd/parsimony/internal/store" ) @@ -46,6 +48,7 @@ func New(s *store.Store, token string, rel Releaser) http.Handler { mux.Handle("POST /api/v1/lists/{id}/members", srv.auth(srv.addMember)) mux.Handle("DELETE /api/v1/lists/{id}/members/{address}", srv.auth(srv.removeMember)) mux.Handle("GET /api/v1/lists/{id}/archive", srv.auth(srv.listArchive)) + mux.Handle("POST /api/v1/lists/{id}/archive/import", srv.auth(srv.importArchive)) mux.Handle("GET /api/v1/archive/{id}", srv.auth(srv.getArchived)) mux.Handle("DELETE /api/v1/archive/{id}", srv.auth(srv.deleteArchived)) mux.Handle("GET /api/v1/queue", srv.auth(srv.getQueue)) @@ -543,3 +546,22 @@ func (s *server) discardQueued(w http.ResponseWriter, r *http.Request) { } w.WriteHeader(http.StatusNoContent) } + +// importArchive reads an mbox from the request body, streaming, and archives it. +// Failed messages are reported in the summary; the request itself succeeds. +func (s *server) importArchive(w http.ResponseWriter, r *http.Request) { + id, ok := listID(w, r) + if !ok { + return + } + if _, err := s.store.GetList(id); err != nil { + storeError(w, err) + return + } + sum, err := archive.ImportMbox(s.store, id, r.Body, time.Now()) + if err != nil { + writeError(w, http.StatusBadRequest, fmt.Sprintf("import stopped after %d message(s): %v", sum.Imported, err)) + return + } + writeJSON(w, http.StatusOK, sum) +} diff --git a/internal/api/api_test.go b/internal/api/api_test.go index 32f21ba..4c22d97 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -522,3 +522,77 @@ func TestQueueRequiresAuth(t *testing.T) { expect(t, do(t, h, "GET", "/api/v1/queue", "", false), 401) expect(t, do(t, h, "DELETE", "/api/v1/queue/1", "", false), 401) } + +const importMbox = "From alice@example.com Tue Jan 2 15:04:05 2024\nFrom: Alice \nSubject: Kickoff\nMessage-ID: <1@x>\nDate: Mon, 01 Jan 2024 10:00:00 +0000\n\nlet's start\n\n" + + "From bob@example.com Tue Jan 2 15:04:05 2024\nFrom: Bob \nSubject: Re: Kickoff\nMessage-ID: <2@x>\nIn-Reply-To: <1@x>\nDate: Mon, 01 Jan 2024 11:00:00 +0000\n\nagreed\n" + +type importJSON struct { + Imported int `json:"imported"` + Duplicates int `json:"duplicates"` + Failed int `json:"failed"` + Errors []string `json:"errors"` +} + +func TestArchiveImport(t *testing.T) { + h, s := queueServer(t) + s.CreateList("dev", "example.org", "Dev") + + rec := do(t, h, "POST", "/api/v1/lists/1/archive/import", importMbox, true) + expect(t, rec, 200) + var sum importJSON + decode(t, rec, &sum) + if sum.Imported != 2 || sum.Duplicates != 0 || sum.Failed != 0 || sum.Errors == nil || len(sum.Errors) != 0 { + t.Fatalf("summary: %+v (errors must be [] not null)", sum) + } + var page archiveJSON + decode(t, do(t, h, "GET", "/api/v1/lists/1/archive", "", true), &page) + if page.Total != 2 || page.Messages[0].Subject != "Re: Kickoff" { + t.Fatalf("archive: %+v", page) + } + if page.Messages[0].ThreadID != page.Messages[1].ThreadID { + t.Fatal("imported replies must be threaded") + } + + rec = do(t, h, "POST", "/api/v1/lists/1/archive/import", importMbox, true) + expect(t, rec, 200) + decode(t, rec, &sum) + if sum.Imported != 0 || sum.Duplicates != 2 { + t.Fatalf("second import: %+v", sum) + } +} + +func TestArchiveImportReportsFailuresWithoutFailingTheRequest(t *testing.T) { + h, s := queueServer(t) + s.CreateList("dev", "example.org", "Dev") + body := importMbox + "\nFrom x@y.z Tue Jan 2 15:04:05 2024\n\nno headers\n" + rec := do(t, h, "POST", "/api/v1/lists/1/archive/import", body, true) + expect(t, rec, 200) + var sum importJSON + decode(t, rec, &sum) + if sum.Imported != 2 || sum.Failed != 1 || len(sum.Errors) != 1 || !strings.Contains(sum.Errors[0], "message 3") { + t.Fatalf("summary: %+v", sum) + } +} + +func TestArchiveImportEdgeCases(t *testing.T) { + h, s := queueServer(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + s.SetPolicies(l.ID, store.PostMembers, store.JoinOpen, store.ArchiveOff) + expect(t, do(t, h, "POST", "/api/v1/lists/99/archive/import", importMbox, true), 404) + expect(t, do(t, h, "POST", "/api/v1/lists/abc/archive/import", importMbox, true), 404) + expect(t, do(t, h, "POST", "/api/v1/lists/1/archive/import", importMbox, false), 401) + + rec := do(t, h, "POST", "/api/v1/lists/1/archive/import", "", true) + expect(t, rec, 200) + var sum importJSON + decode(t, rec, &sum) + if sum.Imported != 0 || sum.Failed != 0 { + t.Fatalf("empty body: %+v", sum) + } + rec = do(t, h, "POST", "/api/v1/lists/1/archive/import", importMbox, true) + expect(t, rec, 200) + decode(t, rec, &sum) + if sum.Imported != 2 { + t.Fatalf("an import works even when the archive policy is off: %+v", sum) + } +} From cc47784fdbe2035704e20f353dfa0c4d46e7bc16 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Mon, 9 Nov 2026 17:15:43 +0100 Subject: [PATCH 4/7] feat(cli): archive import from a file or stdin - Add parsimony archive import , streaming with no overall timeout - Print imported, already present and failed counts; report failing messages and exit 1 - Support --json --- internal/cli/cli.go | 33 +++++++++++++----- internal/cli/cli_test.go | 74 ++++++++++++++++++++++++++++++++++++++++ internal/cli/commands.go | 59 ++++++++++++++++++++++++++++++++ 3 files changed, 158 insertions(+), 8 deletions(-) diff --git a/internal/cli/cli.go b/internal/cli/cli.go index f279966..2dfa020 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -45,6 +45,8 @@ Administration (talks to the REST API): archive ls [--q TEXT] [--limit N] [--offset N] archive show archive rm --yes + archive import add the messages of an mbox file (Mailman export); + safe to repeat, threads are rebuilt queue ls outbound queue queue rm discard one queued delivery @@ -90,7 +92,7 @@ func Run(args []string, io IO, getenv func(string) string) int { "list": {"ls": c.listLs, "add": c.listAdd, "show": c.listShow, "set": c.listSet, "rm": c.listRm}, "member": {"ls": c.memberLs, "add": c.memberAdd, "rm": c.memberRm, "import": c.memberImport}, "held": {"ls": c.heldLs, "approve": c.heldApprove, "discard": c.heldDiscard}, - "archive": {"ls": c.archiveLs, "show": c.archiveShow, "rm": c.archiveRm}, + "archive": {"ls": c.archiveLs, "show": c.archiveShow, "rm": c.archiveRm, "import": c.archiveImport}, "queue": {"ls": c.queueLs, "rm": c.queueRm}, } group, ok := commands[rest[0]] @@ -145,7 +147,11 @@ func (c *cmd) client() (*client, error) { if _, err := url.Parse(base); err != nil { return nil, fmt.Errorf("invalid URL %q", base) } - c.cl = &client{base: strings.TrimRight(base, "/"), token: token, http: &http.Client{Timeout: 30 * time.Second}} + c.cl = &client{ + base: strings.TrimRight(base, "/"), token: token, + http: &http.Client{Timeout: 30 * time.Second}, + upload: &http.Client{}, // big files: no overall timeout + } return c.cl, nil } @@ -176,7 +182,8 @@ func parseArgs(fs *flag.FlagSet, args []string) ([]string, error) { type client struct { base, token string - http *http.Client + http *http.Client // ordinary requests, with a timeout + upload *http.Client // streaming uploads, without one } type apiError struct { @@ -186,25 +193,35 @@ type apiError struct { func (e *apiError) Error() string { return e.message } -// do sends a request and returns the raw response body (nil for 204). +// do sends a JSON request and returns the raw response body (nil for 204). func (cl *client) do(method, path string, body any) (json.RawMessage, error) { var rd io.Reader + ctype := "" if body != nil { b, err := json.Marshal(body) if err != nil { return nil, err } - rd = bytes.NewReader(b) + rd, ctype = bytes.NewReader(b), "application/json" } + return cl.send(cl.http, method, path, rd, ctype) +} + +// uploadTo streams r as the body of a POST, with no overall timeout. +func (cl *client) uploadTo(path string, r io.Reader, contentType string) (json.RawMessage, error) { + return cl.send(cl.upload, "POST", path, r, contentType) +} + +func (cl *client) send(hc *http.Client, method, path string, rd io.Reader, contentType string) (json.RawMessage, error) { req, err := http.NewRequest(method, cl.base+"/api/v1"+path, rd) if err != nil { return nil, err } req.Header.Set("Authorization", "Bearer "+cl.token) - if body != nil { - req.Header.Set("Content-Type", "application/json") + if contentType != "" { + req.Header.Set("Content-Type", contentType) } - resp, err := cl.http.Do(req) + resp, err := hc.Do(req) if err != nil { return nil, fmt.Errorf("cannot reach %s: %w", cl.base, errors.Unwrap(err)) } diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index 34e8c7c..2fb14ee 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -438,3 +438,77 @@ func TestQueue(t *testing.T) { t.Fatalf("pending = %d", st.Pending) } } + +const cliMbox = "From alice@example.com Tue Jan 2 15:04:05 2024\nFrom: Alice \nSubject: Kickoff\nMessage-ID: <1@x>\nDate: Mon, 01 Jan 2024 10:00:00 +0000\n\nlet's start\n\n" + + "From bob@example.com Tue Jan 2 15:04:05 2024\nFrom: Bob \nSubject: Re: Kickoff\nMessage-ID: <2@x>\nIn-Reply-To: <1@x>\nDate: Mon, 01 Jan 2024 11:00:00 +0000\n\nagreed\n" + +func TestArchiveImportFromFileAndStdin(t *testing.T) { + e := newEnv(t) + e.ok(t, "list", "add", "dev@example.org") + path := t.TempDir() + "/dev.mbox" + if err := os.WriteFile(path, []byte(cliMbox), 0o600); err != nil { + t.Fatal(err) + } + out := e.ok(t, "archive", "import", "dev@example.org", path) + if !strings.Contains(out, "Imported 2 messages into dev@example.org") || strings.Contains(out, "failed") || strings.Contains(out, "already") { + t.Fatalf("import: %q", out) + } + l, _ := e.s.FindList("dev@example.org") + if _, total, _ := e.s.ListMessages(l.ID, 10, 0); total != 2 { + t.Fatalf("stored %d", total) + } + + // Again, through stdin: nothing new, and it says so. + code, out, errs := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}, cliMbox, "archive", "import", "1", "-") + if code != 0 || !strings.Contains(out, "Imported 0 messages") || !strings.Contains(out, "2 already present") { + t.Fatalf("second import: %d %q %q", code, out, errs) + } +} + +func TestArchiveImportReportsFailures(t *testing.T) { + e := newEnv(t) + e.ok(t, "list", "add", "dev@example.org") + bad := cliMbox + "\nFrom x@y.z Tue Jan 2 15:04:05 2024\n\nno headers\n" + code, out, errs := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}, bad, "archive", "import", "dev@example.org", "-") + if code != 1 { + t.Fatalf("a failed message must give a non-zero exit status, got %d", code) + } + if !strings.Contains(out, "Imported 2 messages") || !strings.Contains(out, "1 failed") { + t.Fatalf("summary: %q", out) + } + if !strings.Contains(errs, "message 3") || !strings.Contains(errs, "no headers") { + t.Fatalf("the failure is reported: %q", errs) + } +} + +func TestArchiveImportErrorsAndJSON(t *testing.T) { + e := newEnv(t) + e.ok(t, "list", "add", "dev@example.org") + e.fails(t, 1, "no such file", "archive", "import", "dev@example.org", "/nonexistent/x.mbox") + e.fails(t, 1, "no such list", "archive", "import", "ghost@example.org", "/dev/null") + e.fails(t, 2, "usage: parsimony archive import", "archive", "import", "dev@example.org") + _, out, _ := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}, cliMbox, "--json", "archive", "import", "dev@example.org", "-") + var sum map[string]any + if err := json.Unmarshal([]byte(out), &sum); err != nil || sum["imported"].(float64) != 2 { + t.Fatalf("json: %v %q", err, out) + } +} + +func TestLongRunningUploadsAreNotCutByTheShortRequestTimeout(t *testing.T) { + e := newEnv(t) + e.ok(t, "list", "add", "dev@example.org") + var cl *client + c := &cmd{getenv: func(k string) string { + return map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}[k] + }} + cl, err := c.client() + if err != nil { + t.Fatal(err) + } + if cl.http.Timeout == 0 { + t.Fatal("ordinary requests need a timeout") + } + if cl.upload == nil || cl.upload.Timeout != 0 { + t.Fatal("uploads of big archives must not have an overall timeout") + } +} diff --git a/internal/cli/commands.go b/internal/cli/commands.go index cf92e96..2da2c15 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -546,6 +546,65 @@ func (c *cmd) archiveRm(args []string) int { }, "Deleted archived message %d") } +func (c *cmd) archiveImport(args []string) int { + pos, err := parseArgs(newFlagSet("archive import"), args) + if err != nil { + return c.usageError("%v", err) + } + if len(pos) != 2 { + return c.usageError("usage: parsimony archive import ") + } + l, err := c.resolveList(pos[0]) + if err != nil { + return c.fail(err) + } + var in io.Reader = c.io.In + if pos[1] != "-" { + f, err := os.Open(pos[1]) + if err != nil { + return c.fail(err) + } + defer f.Close() + in = f + } + cl, _ := c.client() + raw, err := cl.uploadTo(fmt.Sprintf("/lists/%d/archive/import", l.ID), in, "application/mbox") + if err != nil { + return c.fail(err) + } + var sum struct { + Imported int `json:"imported"` + Duplicates int `json:"duplicates"` + Failed int `json:"failed"` + Errors []string `json:"errors"` + } + if err := json.Unmarshal(raw, &sum); err != nil { + return c.fail(err) + } + if c.json { + c.printJSON(raw) + } else { + msg := fmt.Sprintf("Imported %s into %s", plural(sum.Imported, "message", "messages"), l.Address) + if sum.Duplicates > 0 { + msg += fmt.Sprintf(", %d already present", sum.Duplicates) + } + if sum.Failed > 0 { + msg += fmt.Sprintf(", %d failed", sum.Failed) + } + fmt.Fprintln(c.io.Out, msg) + } + for _, e := range sum.Errors { + fmt.Fprintln(c.io.Err, e) + } + if sum.Failed > len(sum.Errors) { + fmt.Fprintf(c.io.Err, "... and %d more\n", sum.Failed-len(sum.Errors)) + } + if sum.Failed > 0 { + return 1 + } + return 0 +} + // --- queue -------------------------------------------------------------------------- func (c *cmd) queueLs(args []string) int { From d8cc5ed68a1ec8b49e86b617d14120be19b6e14c Mon Sep 17 00:00:00 2001 From: tgiachi Date: Mon, 9 Nov 2026 19:45:48 +0100 Subject: [PATCH 5/7] feat(admin): import an mbox from the Archive tab - Add upload of an mbox file with a summary of imported, already present and failed messages - List the first failing messages and refresh the archive - Stream the file as the raw request body --- admin/src/lib/api.test.ts | 13 ++++++ admin/src/lib/api.ts | 14 +++++- admin/src/tabs/ArchiveTab.test.tsx | 74 ++++++++++++++++++++++++++++++ admin/src/tabs/ArchiveTab.tsx | 67 +++++++++++++++++++++++++-- admin/src/test/fakeApi.ts | 37 ++++++++++++++- 5 files changed, 198 insertions(+), 7 deletions(-) diff --git a/admin/src/lib/api.test.ts b/admin/src/lib/api.test.ts index c8622e6..02124df 100644 --- a/admin/src/lib/api.test.ts +++ b/admin/src/lib/api.test.ts @@ -159,4 +159,17 @@ describe('endpoints', () => { ['GET', '/api/v1/queue'], ]) }) + + it('uploads an mbox as the raw body with its own content type', async () => { + mockFetch(respond(200, { imported: 2, duplicates: 0, failed: 0, errors: [] })) + const file = new File(['From a@x.com Tue Jan 2 15:04:05 2024\nSubject: x\n\nbody\n'], 'dev.mbox') + const sum = await api.archive.import(7, file) + expect(sum).toEqual({ imported: 2, duplicates: 0, failed: 0, errors: [] }) + expect(calls[0].url).toBe('/api/v1/lists/7/archive/import') + expect(calls[0].init.method).toBe('POST') + expect(calls[0].init.body).toBe(file) // streamed as is, not JSON-encoded + const headers = calls[0].init.headers as Record + expect(headers['Content-Type']).toBe('application/mbox') + expect(headers.Authorization).toBe('Bearer secret-token') + }) }) diff --git a/admin/src/lib/api.ts b/admin/src/lib/api.ts index 23666fb..a33e798 100644 --- a/admin/src/lib/api.ts +++ b/admin/src/lib/api.ts @@ -42,6 +42,13 @@ export interface Message { body?: string } +export interface ImportSummary { + imported: number + duplicates: number + failed: number + errors: string[] +} + export interface ArchivePage { total: number messages: Message[] @@ -90,14 +97,16 @@ async function request(method: string, path: string, body?: unknown): Promise const headers: Record = {} const token = session.get() if (token) headers.Authorization = `Bearer ${token}` - if (body !== undefined) headers['Content-Type'] = 'application/json' + // A file is streamed as is; everything else is JSON. + const raw = body instanceof Blob + if (body !== undefined) headers['Content-Type'] = raw ? 'application/mbox' : 'application/json' let res: Response try { res = await fetch(`/api/v1${path}`, { method, headers, - body: body === undefined ? undefined : JSON.stringify(body), + body: body === undefined ? undefined : raw ? body : JSON.stringify(body), }) } catch { throw new ApiError(0, 'Cannot reach the server') @@ -159,6 +168,7 @@ export const api = { const s = qs.toString() return request('GET', `/lists/${listId}/archive${s ? `?${s}` : ''}`) }, + import: (listId: number, file: Blob) => request('POST', `/lists/${listId}/archive/import`, file), get: (id: number) => request('GET', `/archive/${id}`), remove: (id: number) => request('DELETE', `/archive/${id}`), }, diff --git a/admin/src/tabs/ArchiveTab.test.tsx b/admin/src/tabs/ArchiveTab.test.tsx index b255286..20360af 100644 --- a/admin/src/tabs/ArchiveTab.test.tsx +++ b/admin/src/tabs/ArchiveTab.test.tsx @@ -171,4 +171,78 @@ describe('archive tab', () => { expect(state.archive[1]).toHaveLength(2) }) }) + + describe('importing an mbox', () => { + const entry = (n: number) => + `From user${n}@x.com Tue Jan 2 15:04:05 2024\nFrom: User ${n} \nSubject: Imported ${n}\nMessage-ID: \n\nbody ${n}\n\n` + const mboxFile = (...ns: number[]) => new File([ns.map(entry).join('')], 'dev.mbox', { type: 'application/mbox' }) + + it('uploads the file, shows what happened and lists the new messages', async () => { + const { user, state } = await setup([]) + await screen.findByText(/no messages archived yet/i) + await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1, 2, 3)) + + const status = await screen.findByRole('status') + expect(status).toHaveTextContent('Imported 3 messages') + expect(state.importTypes).toEqual(['application/mbox']) + expect(state.archive[1]).toHaveLength(3) + expect(await screen.findByRole('button', { name: 'Imported 3' })).toBeInTheDocument() + expect(screen.getByRole('heading', { name: 'Archive (3)', level: 2 })).toBeInTheDocument() + }) + + it('says when messages were already there', async () => { + const { user } = await setup([]) + await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1, 2)) + await screen.findByText(/Imported 2 messages/) + await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1, 2, 3)) + const status = await screen.findByText(/Imported 1 message\b/) + expect(status).toHaveTextContent('2 already present') + }) + + it('lists the messages that could not be read', async () => { + const { user } = await setup([]) + const file = new File([entry(1) + 'From x@y.z Tue Jan 2 15:04:05 2024\n\nno headers here\n'], 'bad.mbox') + await user.upload(screen.getByLabelText(/mbox file/i), file) + const status = await screen.findByRole('status') + expect(status).toHaveTextContent('1 failed') + expect(within(status).getByText(/message 2: parse message: no headers/)).toBeInTheDocument() + }) + + it('shows a server error', async () => { + const { user, state } = await setup([]) + state.fail['POST /lists/1/archive/import'] = { status: 500, error: 'disk full' } + await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1)) + expect(await screen.findByRole('alert')).toHaveTextContent('disk full') + expect(screen.queryByRole('status')).not.toBeInTheDocument() + }) + + it('shows that it is working, and cannot be started twice', async () => { + let release!: () => void + const { user, state } = await setup([]) + state.hold = new Promise((resolve) => (release = resolve)) + await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1)) + const busy = await screen.findByRole('button', { name: /importing/i }) + expect(busy).toBeDisabled() + release() + expect(await screen.findByRole('status')).toHaveTextContent('Imported 1 message') + expect(screen.getByRole('button', { name: /^import mbox$/i })).toBeEnabled() + }) + + it('opens the file chooser from the button and accepts mbox-like files', async () => { + await setup([]) + const input = screen.getByLabelText(/mbox file/i) as HTMLInputElement + expect(input.type).toBe('file') + expect(input.accept).toMatch(/\.mbox/) + expect(screen.getByRole('button', { name: /^import mbox$/i })).toBeInTheDocument() + }) + + it('can import the same file again after a first attempt (the input is reset)', async () => { + const { user, state } = await setup([]) + const file = mboxFile(1) + await user.upload(screen.getByLabelText(/mbox file/i), file) + await screen.findByText(/Imported 1 message/) + await user.upload(screen.getByLabelText(/mbox file/i), file) + await waitFor(() => expect(state.importTypes).toHaveLength(2)) + }) + }) }) diff --git a/admin/src/tabs/ArchiveTab.tsx b/admin/src/tabs/ArchiveTab.tsx index 611ffa6..efcb1f8 100644 --- a/admin/src/tabs/ArchiveTab.tsx +++ b/admin/src/tabs/ArchiveTab.tsx @@ -1,22 +1,48 @@ import { keepPreviousData, useMutation, useQuery, useQueryClient } from '@tanstack/react-query' -import { useState, type FormEvent } from 'react' +import { useRef, useState, type ChangeEvent, type FormEvent } from 'react' import { useOutletContext } from 'react-router' import { toast } from 'sonner' import { Button } from '@/components/ui/button' import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from '@/components/ui/dialog' import { Input } from '@/components/ui/input' import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table' -import { api, type List } from '@/lib/api' +import { api, type ImportSummary, type List } from '@/lib/api' import { formatBytes, formatDate } from '@/lib/format' const PAGE_SIZE = 25 +function plural(n: number, one: string, many: string) { + return `${n} ${n === 1 ? one : many}` +} + export function ArchiveTab() { const list = useOutletContext() const [input, setInput] = useState('') const [q, setQ] = useState('') const [page, setPage] = useState(0) const [openId, setOpenId] = useState(null) + const queryClient = useQueryClient() + const fileInput = useRef(null) + const [imported, setImported] = useState(null) + const [importError, setImportError] = useState(null) + + const importMbox = useMutation({ + mutationFn: (file: File) => api.archive.import(list.id, file), + onSuccess: async (summary) => { + setImported(summary) + await queryClient.invalidateQueries({ queryKey: ['archive', list.id] }) + }, + onError: (e: Error) => setImportError(e.message), + }) + + function chooseFile(e: ChangeEvent) { + const file = e.target.files?.[0] + e.target.value = '' // lets the same file be chosen again + if (!file) return + setImported(null) + setImportError(null) + importMbox.mutate(file) + } const archive = useQuery({ queryKey: ['archive', list.id, q, page], @@ -43,7 +69,42 @@ export function ArchiveTab() { return (
-

Archive{archive.data && !q ? ` (${total})` : ''}

+
+

Archive{archive.data && !q ? ` (${total})` : ''}

+ + +
+ {importError && ( +

+ {importError} +

+ )} + {imported && ( +
+

+ Imported {plural(imported.imported, 'message', 'messages')} + {imported.duplicates > 0 && `, ${imported.duplicates} already present`} + {imported.failed > 0 && `, ${imported.failed} failed`} +

+ {imported.errors.length > 0 && ( +
    + {imported.errors.map((e) => ( +
  • {e}
  • + ))} + {imported.failed > imported.errors.length &&
  • … and {imported.failed - imported.errors.length} more
  • } +
+ )} +
+ )} {list.archive_policy === 'off' && (

Archiving is off for this list: new messages are not stored. Set Archive to private or public in Settings. diff --git a/admin/src/test/fakeApi.ts b/admin/src/test/fakeApi.ts index 4355086..fd93424 100644 --- a/admin/src/test/fakeApi.ts +++ b/admin/src/test/fakeApi.ts @@ -11,6 +11,10 @@ export interface FakeState { queue: QueueItem[] /** Like the server, the queue listing is capped. */ queueLimit: number + /** Content types seen on archive imports. */ + importTypes: string[] + /** When set, archive imports wait for it (to look at the busy state). */ + hold?: Promise nextId: number /** Every request seen, as "METHOD /path". */ calls: string[] @@ -19,7 +23,7 @@ export interface FakeState { } export function newState(overrides: Partial = {}): FakeState { - return { token: 'good', lists: [], members: {}, held: {}, archive: {}, queue: [], queueLimit: 100, nextId: 100, calls: [], fail: {}, ...overrides } + return { token: 'good', lists: [], members: {}, held: {}, archive: {}, queue: [], queueLimit: 100, importTypes: [], nextId: 100, calls: [], fail: {}, ...overrides } } export function aList(over: Partial = {}): List { @@ -62,7 +66,8 @@ export function installFakeApi(state: FakeState) { delete state.fail[key] return err(f.status, f.error) } - const body = init.body ? JSON.parse(init.body as string) : {} + // Files (archive imports) are not JSON. + const body = typeof init.body === 'string' && init.body ? JSON.parse(init.body) : {} let m: RegExpMatchArray | null if (path === '/queue' && method === 'GET') { @@ -132,6 +137,34 @@ export function installFakeApi(state: FakeState) { } return err(404, 'not found') } + if ((m = path.match(/^\/lists\/(\d+)\/archive\/import$/)) && method === 'POST') { + const id = Number(m[1]) + if (!state.lists.some((l) => l.id === id)) return err(404, 'not found') + if (state.hold) await state.hold + state.importTypes.push((init.headers as Record)['Content-Type'] ?? '') + const text = typeof init.body === 'string' ? init.body : await (init.body as Blob).text() + const messages = text.split(/^From \S+ .*$/m).slice(1) + const archive = (state.archive[id] ??= []) + let imported = 0 + let duplicates = 0 + const errors: string[] = [] + messages.forEach((raw, i) => { + const subject = /^Subject: (.*)$/m.exec(raw)?.[1] + const messageId = /^Message-ID: (.*)$/m.exec(raw)?.[1] + if (!subject || !messageId) { + errors.push(`message ${i + 1}: parse message: no headers`) + return + } + if (archive.some((a) => a.message_id === messageId)) { + duplicates++ + return + } + const nid = state.nextId++ + archive.push({ id: nid, thread_id: nid, message_id: messageId, subject, sender: 'Imported', date: '2024-01-01T00:00:00Z', size: raw.length }) + imported++ + }) + return json(200, { imported, duplicates, failed: errors.length, errors }) + } if ((m = path.match(/^\/lists\/(\d+)\/archive$/)) && method === 'GET') { const id = Number(m[1]) const q = (url.searchParams.get('q') ?? '').toLowerCase() From a170204ecb9053cb39e31d41a040658fe7519e2c Mon Sep 17 00:00:00 2001 From: tgiachi Date: Mon, 9 Nov 2026 19:50:40 +0100 Subject: [PATCH 6/7] test: close gaps found by mutation checks on the archive import - Cover the CRLF separator line, cycle roots entered from outside, linear rethreading of long chains - Cover a read error mid-import, the "and N more" lines, and the mbox content type - Cover clearing the previous result and the capped error list in the UI --- admin/src/tabs/ArchiveTab.test.tsx | 38 ++++++++++++++++++++++- internal/api/api_test.go | 33 ++++++++++++++++++++ internal/cli/cli_test.go | 40 ++++++++++++++++++++++++ internal/mbox/mbox_test.go | 3 ++ internal/store/archive_test.go | 49 ++++++++++++++++++++++++++++++ 5 files changed, 162 insertions(+), 1 deletion(-) diff --git a/admin/src/tabs/ArchiveTab.test.tsx b/admin/src/tabs/ArchiveTab.test.tsx index 20360af..49f6d50 100644 --- a/admin/src/tabs/ArchiveTab.test.tsx +++ b/admin/src/tabs/ArchiveTab.test.tsx @@ -1,5 +1,5 @@ import { screen, waitFor, within } from '@testing-library/react' -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import type { Message } from '@/lib/api' import { aList, installFakeApi, newState } from '@/test/fakeApi' import { renderApp } from '@/test/render' @@ -244,5 +244,41 @@ describe('archive tab', () => { await user.upload(screen.getByLabelText(/mbox file/i), file) await waitFor(() => expect(state.importTypes).toHaveLength(2)) }) + + it('forgets the previous result when a new import starts', async () => { + const { user, state } = await setup([]) + state.fail['POST /lists/1/archive/import'] = { status: 500, error: 'disk full' } + await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1)) + expect(await screen.findByRole('alert')).toHaveTextContent('disk full') + + await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1)) + expect(await screen.findByRole('status')).toHaveTextContent('Imported 1 message') + expect(screen.queryByRole('alert')).not.toBeInTheDocument() // the old error is gone + + state.fail['POST /lists/1/archive/import'] = { status: 500, error: 'again' } + await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(2)) + expect(await screen.findByRole('alert')).toHaveTextContent('again') + expect(screen.queryByRole('status')).not.toBeInTheDocument() // the old summary is gone + }) + + it('says how many more failures there are than the server listed', async () => { + const { user } = await setup([]) + const real = fetch + vi.stubGlobal('fetch', vi.fn((url: string, init?: RequestInit) => { + if (url.endsWith('/archive/import')) { + const errors = Array.from({ length: 20 }, (_, i) => `message ${i + 1}: parse message: no headers`) + return Promise.resolve(new Response(JSON.stringify({ imported: 0, duplicates: 0, failed: 25, errors }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + })) + } + return real(url, init) + })) + await user.upload(screen.getByLabelText(/mbox file/i), mboxFile(1)) + const status = await screen.findByRole('status') + expect(status).toHaveTextContent('25 failed') + expect(within(status).getByText(/and 5 more/)).toBeInTheDocument() + expect(within(status).getAllByRole('listitem')).toHaveLength(21) + }) }) }) diff --git a/internal/api/api_test.go b/internal/api/api_test.go index 4c22d97..1e58c06 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -3,6 +3,7 @@ package api import ( "context" "encoding/json" + "errors" "fmt" "net/http" "net/http/httptest" @@ -596,3 +597,35 @@ func TestArchiveImportEdgeCases(t *testing.T) { t.Fatalf("an import works even when the archive policy is off: %+v", sum) } } + +type failingBody struct { + data string + done bool +} + +func (f *failingBody) Read(p []byte) (int, error) { + if !f.done { + f.done = true + return copy(p, f.data), nil + } + return 0, errors.New("connection reset by peer") +} + +func TestArchiveImportStoppedByAReadErrorIsAnErrorResponse(t *testing.T) { + h, s := queueServer(t) + s.CreateList("dev", "example.org", "Dev") + req := httptest.NewRequest("POST", "/api/v1/lists/1/archive/import", &failingBody{data: importMbox}) + req.Header.Set("Authorization", "Bearer "+token) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + expect(t, rec, 400) + var e struct{ Error string } + decode(t, rec, &e) + if !strings.Contains(e.Error, "connection reset") || !strings.Contains(e.Error, "1 message") { + t.Fatalf("the error says what happened and how far it got: %q", e.Error) + } + l, _ := s.FindList("dev@example.org") + if _, total, _ := s.ListMessages(l.ID, 10, 0); total != 1 { + t.Fatalf("the first message was complete and is kept, got %d", total) + } +} diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index 2fb14ee..f64aece 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "fmt" + "io" "net/http" "net/http/httptest" "os" @@ -512,3 +513,42 @@ func TestLongRunningUploadsAreNotCutByTheShortRequestTimeout(t *testing.T) { t.Fatal("uploads of big archives must not have an overall timeout") } } + +func TestArchiveImportSaysHowManyMoreFailuresThereAre(t *testing.T) { + e := newEnv(t) + e.ok(t, "list", "add", "dev@example.org") + var b strings.Builder + for i := 0; i < 25; i++ { + b.WriteString("From x@y.z Tue Jan 2 15:04:05 2024\n\nno headers\n\n") + } + code, out, errs := e.runEnv(t, map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": e.srv.URL}, b.String(), "archive", "import", "dev@example.org", "-") + if code != 1 || !strings.Contains(out, "25 failed") { + t.Fatalf("%d %q", code, out) + } + if !strings.Contains(errs, "... and 5 more") { + t.Fatalf("only 20 are listed, the rest is counted: %q", errs) + } +} + +func TestArchiveImportSendsTheFileAsAnMbox(t *testing.T) { + e := newEnv(t) + e.ok(t, "list", "add", "dev@example.org") + var contentType, body string + spy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/archive/import") { + contentType = r.Header.Get("Content-Type") + b, _ := io.ReadAll(r.Body) + body = string(b) + r.Body = io.NopCloser(strings.NewReader(body)) + } + e.srv.Config.Handler.ServeHTTP(w, r) + })) + defer spy.Close() + vars := map[string]string{"PARSIMONY_API_TOKEN": token, "PARSIMONY_URL": spy.URL} + if code, _, errs := e.runEnv(t, vars, cliMbox, "archive", "import", "dev@example.org", "-"); code != 0 { + t.Fatalf("%d %s", code, errs) + } + if contentType != "application/mbox" || body != cliMbox { + t.Fatalf("content type %q, body unchanged: %v", contentType, body == cliMbox) + } +} diff --git a/internal/mbox/mbox_test.go b/internal/mbox/mbox_test.go index c508f43..a39e7e5 100644 --- a/internal/mbox/mbox_test.go +++ b/internal/mbox/mbox_test.go @@ -104,6 +104,9 @@ func TestCRLFAndMissingFinalNewline(t *testing.T) { if len(ms) != 1 || !strings.HasSuffix(string(ms[0].Data), "body without newline") || !strings.Contains(string(ms[0].Data), "Subject: s\r\n") { t.Fatalf("%q", ms[0].Data) } + if string(readAll(t, "From a@x.com Tue Jan 2 15:04:05 2024\r\nSubject: one\r\n\r\nb\r\n\r\nFrom b@x.com Tue Jan 2 15:04:05 2024\r\nSubject: two\r\n\r\nb2\r\n", 0)[0].Data) != "Subject: one\r\n\r\nb\r\n" { + t.Fatal("the CRLF blank line before a separator belongs to the separator") + } crlf := "From a@x.com Tue Jan 2 15:04:05 2024\r\nSubject: one\r\n\r\nb\r\n\r\nFrom b@x.com Tue Jan 2 15:04:05 2024\r\nSubject: two\r\n\r\nb2\r\n" if got := readAll(t, crlf, 0); len(got) != 2 { t.Fatalf("CRLF separators: %d", len(got)) diff --git a/internal/store/archive_test.go b/internal/store/archive_test.go index 1ee9133..c8fb809 100644 --- a/internal/store/archive_test.go +++ b/internal/store/archive_test.go @@ -2,6 +2,7 @@ package store import ( "errors" + "fmt" "strings" "testing" "time" @@ -383,3 +384,51 @@ func TestRethreadOnlyTouchesTheGivenList(t *testing.T) { t.Fatal("list b's child must now follow its parent") } } + +func TestRethreadCycleRootIsTheSmallestIdEvenWhenEnteredFromElsewhere(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + // ids: x=1 replies to c; b=2 replies to c; c=3 replies to b. The cycle is {b, c}, entered from x at c, + // which is not its smallest id. + x := msg("", "X", "x") + x.InReplyTo = "" + b := msg("", "B", "b") + b.InReplyTo = "" + c := msg("", "C", "c") + c.InReplyTo = "" + ix, _, _ := s.ArchiveMessage(l.ID, x) + ib, _, _ := s.ArchiveMessage(l.ID, b) + ic, _, _ := s.ArchiveMessage(l.ID, c) + if _, err := s.Rethread(l.ID); err != nil { + t.Fatal(err) + } + want := min(ib, ic) + for _, id := range []int64{ix, ib, ic} { + if got := threadOf(t, s, id); got != want { + t.Fatalf("message %d: thread %d, want %d (the smallest id of the cycle)", id, got, want) + } + } +} + +func TestRethreadIsLinearOnLongChains(t *testing.T) { + s := newTestStore(t) + l, _ := s.CreateList("dev", "example.org", "Dev") + const n = 20000 + ms := make([]Message, n) + for i := range ms { + ms[i] = msg(fmt.Sprintf("", i), "S", "b") + if i > 0 { + ms[i].InReplyTo = fmt.Sprintf("", i-1) + } + } + if _, err := s.ArchiveBatch(l.ID, ms); err != nil { + t.Fatal(err) + } + done := make(chan struct{}) + go func() { s.Rethread(l.ID); close(done) }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Rethread is not linear: a 20000-message chain took more than 5 seconds") + } +} From b777acc1547d596df2552f80dfac0be03876e746 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Mon, 9 Nov 2026 19:52:14 +0100 Subject: [PATCH 7/7] feat: end-to-end test and docs for the archive import - Import a Mailman-style mbox through the CLI inside the container: encoded headers, multipart, escaped From lines, threading, idempotence - Check accent-insensitive search and the public page hiding addresses - Document the migration from Mailman in the README and the endpoint in OpenAPI --- README.md | 18 +++++++++++++ docs/openapi.yaml | 29 +++++++++++++++++++++ e2e/e2e_test.go | 64 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+) diff --git a/README.md b/README.md index df253b3..97ae3e0 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,7 @@ parsimony member import dev@example.org subscribers.txt # "email[,name]" per l parsimony held ls dev@example.org parsimony held approve 12 parsimony archive ls dev@example.org --q "deploy" +parsimony archive import dev@example.org dev.mbox # an mbox file or "-" for stdin parsimony queue ls parsimony --json list ls # JSON for scripts parsimony help @@ -71,6 +72,23 @@ Exit status: 0 success, 1 failure (including lines that failed in `member import `parsimony serve -config FILE` runs the daemon; the old `parsimony -config FILE` form still works. In the container: `docker exec parsimony /parsimony list ls`. +## Migrating from Mailman + +Both Mailman 2 and Mailman 3 (HyperKitty) can export a list's history as an mbox file. + +``` +parsimony list add dev@example.org --title "Developers" +parsimony member import dev@example.org subscribers.txt # "email[,name]" per line +parsimony archive import dev@example.org dev.mbox +``` + +The import reads standard mbox (a message starts at a `From addr Tue Jan 2 15:04:05 2024` line, `>From ` escapes are +restored), decodes encoded headers, charsets and multipart bodies, and stores the plain text. It is safe to repeat: a +message already in the archive (same `Message-ID`) is skipped. Threads are rebuilt from `In-Reply-To` whatever the order +of the messages. Messages that cannot be read are listed and counted, and the exit status is 1; the others are kept. +It works whatever the list's archive policy is (the policy only governs live mail). The admin UI has the same import on +the Archive tab. Search is accent-insensitive: `caffe` finds "Caffè". + ## Docker The image runs as an unprivileged user (uid 65532) and works with a read-only root filesystem. diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 52395a0..9b6a132 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -275,3 +275,32 @@ paths: responses: '204': {description: Discarded} '404': {$ref: '#/components/responses/Error'} + /lists/{id}/archive/import: + parameters: + - {name: id, in: path, required: true, schema: {type: integer}} + post: + summary: Import an mbox file into the archive + description: > + The request body is the mbox file itself (any content type, `application/mbox` by convention), streamed. + Idempotent: a message already archived (same Message-ID) is counted as a duplicate. Threads are rebuilt at the end. + Works whatever the list's archive policy. Messages that cannot be read are reported in the summary and do not + fail the request; a read error while receiving the body does (400), keeping the messages read so far. + requestBody: + required: true + content: + application/mbox: + schema: {type: string, format: binary} + responses: + '200': + description: Summary + content: + application/json: + schema: + type: object + properties: + imported: {type: integer} + duplicates: {type: integer} + failed: {type: integer} + errors: {type: array, items: {type: string}, description: The first 20 failures} + '400': {$ref: '#/components/responses/Error'} + '404': {$ref: '#/components/responses/Error'} diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 91d2a42..714d7eb 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -544,6 +544,70 @@ func TestWholePath(t *testing.T) { } }) + t.Run("a Mailman-style mbox can be imported and browsed", func(t *testing.T) { + mbox := "From MAILER-DAEMON Mon Jan 1 10:00:00 2024\n" + + "From: =?UTF-8?Q?Jos=C3=A9_Garc=C3=ADa?= \nSubject: =?UTF-8?Q?Caff=C3=A8_a_las_nueve?=\nMessage-ID: \nDate: Mon, 01 Jan 2024 10:00:00 +0000\nMIME-Version: 1.0\nContent-Type: multipart/alternative; boundary=BB\n\n" + + "--BB\nContent-Type: text/plain; charset=utf-8\nContent-Transfer-Encoding: quoted-printable\n\nCiao a tutti, scrivetemi a jose@old-list.example\n>From the old archive: caff=C3=A8!\n--BB\nContent-Type: text/html\n\n

html copy

\n--BB--\n\n" + + "From carol@members.test Tue Jan 2 09:00:00 2024\n" + + "From: Carol \nSubject: Re: Caff\xc3\xa8\nMessage-ID: \nIn-Reply-To: \nDate: Tue, 02 Jan 2024 09:00:00 +0000\n\nsee you there\n" + imp := exec.Command("docker", "exec", "-i", parsimonyC, "/parsimony", "archive", "import", list, "-") + imp.Stdin = strings.NewReader(mbox) + out, err := imp.CombinedOutput() + if err != nil || !strings.Contains(string(out), "Imported 2 messages") { + t.Fatalf("archive import: %v\n%s", err, out) + } + // Importing again adds nothing. + again := exec.Command("docker", "exec", "-i", parsimonyC, "/parsimony", "archive", "import", list, "-") + again.Stdin = strings.NewReader(mbox) + out, err = again.CombinedOutput() + if err != nil || !strings.Contains(string(out), "Imported 0 messages") || !strings.Contains(string(out), "2 already present") { + t.Fatalf("second import: %v\n%s", err, out) + } + + var page struct { + Messages []struct { + ID int64 `json:"id"` + ThreadID int64 `json:"thread_id"` + Subject string `json:"subject"` + Sender string `json:"sender"` + } `json:"messages"` + } + mustCall(t, 200, "GET", "/api/v1/lists/1/archive?q=nueve", "", &page) + if len(page.Messages) != 1 || page.Messages[0].Subject != "Caffè a las nueve" || page.Messages[0].Sender != "José García" { + t.Fatalf("encoded headers must be decoded: %+v", page.Messages) + } + root := page.Messages[0] + var msg struct { + Body string `json:"body"` + } + mustCall(t, 200, "GET", fmt.Sprintf("/api/v1/archive/%d", root.ID), "", &msg) + if !strings.Contains(msg.Body, "caffè!") || !strings.Contains(msg.Body, "\nFrom the old archive") || strings.Contains(msg.Body, "html copy") { + t.Fatalf("body: %q", msg.Body) + } + var all struct { + Messages []struct { + ThreadID int64 `json:"thread_id"` + Subject string `json:"subject"` + } `json:"messages"` + } + mustCall(t, 200, "GET", "/api/v1/lists/1/archive?q=caffe", "", &all) + if len(all.Messages) != 2 || all.Messages[0].ThreadID != all.Messages[1].ThreadID { + t.Fatalf("the reply must be in its parent's thread: %+v", all.Messages) + } + + // The list's archive is public: imported messages show up on the web, with addresses hidden. + resp, err := http.Get(fmt.Sprintf("%s/archive/%s/%d", webBase, list, root.ID)) + if err != nil { + t.Fatal(err) + } + b, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != 200 || !strings.Contains(string(b), "José García") || !strings.Contains(string(b), "jose@…") || + strings.Contains(string(b), "jose@old-list.example") { + t.Fatalf("public page: %d\n%.700s", resp.StatusCode, b) + } + }) + t.Run("the queue ends empty", func(t *testing.T) { waitQueueEmpty(t) })