From 45a61ef29459464ad2ad744b7aa587c3da186e90 Mon Sep 17 00:00:00 2001 From: tgiachi Date: Mon, 9 Nov 2026 17:14:46 +0100 Subject: [PATCH] feat(archive): import messages from an mbox - Add ImportMbox: batches of 200, idempotent by Message-ID, threads rebuilt at the end - Date fallbacks: Date header, then the From line, then the import time - Report failed and oversized messages without stopping, keep progress on a read error - Import regardless of the list's archive policy --- internal/archive/import.go | 106 +++++++++++++ internal/archive/import_test.go | 254 ++++++++++++++++++++++++++++++++ 2 files changed, 360 insertions(+) create mode 100644 internal/archive/import.go create mode 100644 internal/archive/import_test.go diff --git a/internal/archive/import.go b/internal/archive/import.go new file mode 100644 index 0000000..ce2a4c5 --- /dev/null +++ b/internal/archive/import.go @@ -0,0 +1,106 @@ +package archive + +import ( + "errors" + "fmt" + "io" + "time" + + "codeberg.org/squidd/parsimony/internal/mbox" + "codeberg.org/squidd/parsimony/internal/store" +) + +const ( + // MaxImportMessage is the largest message accepted from an mbox, in bytes. + MaxImportMessage = 25 << 20 + // MaxReportedErrors caps the error messages kept in a summary; Failed still counts every one. + MaxReportedErrors = 20 + + importBatch = 200 // messages per transaction +) + +// ImportSummary reports what an import did. +type ImportSummary struct { + Imported int `json:"imported"` // new messages stored + Duplicates int `json:"duplicates"` // already in the archive (same Message-ID) + Failed int `json:"failed"` // unreadable or too large + Errors []string `json:"errors"` // the first MaxReportedErrors failures +} + +// ImportMbox archives every message of an mbox into a list. It is idempotent +// (a message is identified by its Message-ID) and works whatever the list's +// archive policy is: the policy only governs live mail. Threads are rebuilt at +// the end, so the order of the messages in the file does not matter. now is the +// date of last resort for a message with neither a Date header nor a dated +// "From " line. On a read error the messages read so far are kept and returned +// in the summary together with the error. +func ImportMbox(st *store.Store, listID int64, r io.Reader, now time.Time) (ImportSummary, error) { + sum := ImportSummary{Errors: []string{}} + var batch []store.Message + + fail := func(index int, err error) { + sum.Failed++ + if len(sum.Errors) < MaxReportedErrors { + sum.Errors = append(sum.Errors, fmt.Sprintf("message %d: %v", index, err)) + } + } + flush := func() error { + results, err := st.ArchiveBatch(listID, batch) + if err != nil { + return err + } + for _, res := range results { + if res.Created { + sum.Imported++ + } else { + sum.Duplicates++ + } + } + batch = batch[:0] + return nil + } + finish := func(readErr error) (ImportSummary, error) { + if err := flush(); err != nil { + return sum, err + } + if sum.Imported > 0 { + if _, err := st.Rethread(listID); err != nil { + return sum, err + } + } + return sum, readErr + } + + rd := mbox.NewReader(r, MaxImportMessage) + for { + m, err := rd.Next() + if err == io.EOF { + return finish(nil) + } + if errors.Is(err, mbox.ErrTooLarge) { + fail(m.Index, fmt.Errorf("too large (limit %d MiB)", MaxImportMessage>>20)) + continue + } + if err != nil { + return finish(fmt.Errorf("read mbox: %w", err)) + } + fallback := now + if d, ok := m.Date(); ok { + fallback = d + } + en, err := Parse(m.Data, fallback) + if err != nil { + fail(m.Index, err) + continue + } + batch = append(batch, store.Message{ + MessageID: en.MessageID, InReplyTo: en.InReplyTo, Subject: en.Subject, + Sender: en.Sender, Date: en.Date, Body: en.Body, + }) + if len(batch) >= importBatch { + if err := flush(); err != nil { + return sum, err + } + } + } +} diff --git a/internal/archive/import_test.go b/internal/archive/import_test.go new file mode 100644 index 0000000..6526227 --- /dev/null +++ b/internal/archive/import_test.go @@ -0,0 +1,254 @@ +package archive + +import ( + "errors" + "fmt" + "strings" + "testing" + "time" + + "codeberg.org/squidd/parsimony/internal/store" +) + +var importNow = time.Date(2031, 5, 6, 7, 8, 9, 0, time.UTC) + +func newList(t *testing.T) (*store.Store, store.List) { + t.Helper() + s, err := store.Open(":memory:") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + l, _ := s.CreateList("dev", "example.org", "Dev") + return s, l +} + +func mboxMsg(from, subject, id, replyTo, date, body string) string { + addr := from + if i := strings.Index(from, "<"); i >= 0 { + addr = strings.Trim(from[i:], "<>") + } + h := "From " + addr + " Tue Jan 2 15:04:05 2024\n" + h += "From: " + from + "\nSubject: " + subject + "\n" + if id != "" { + h += "Message-ID: " + id + "\n" + } + if replyTo != "" { + h += "In-Reply-To: " + replyTo + "\n" + } + if date != "" { + h += "Date: " + date + "\n" + } + return h + "\n" + body + "\n\n" +} + +func TestImportMbox(t *testing.T) { + s, l := newList(t) + mbox := mboxMsg("Alice ", "Kickoff", "<1@x>", "", "Mon, 01 Jan 2024 10:00:00 +0000", "let's start") + + mboxMsg("Bob ", "Re: Kickoff", "<2@x>", "<1@x>", "Mon, 01 Jan 2024 11:00:00 +0000", "agreed") + + mboxMsg("Carol ", "Lunch", "<3@x>", "", "Tue, 02 Jan 2024 12:00:00 +0000", "pizza?") + sum, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow) + if err != nil { + t.Fatal(err) + } + if sum.Imported != 3 || sum.Duplicates != 0 || sum.Failed != 0 || len(sum.Errors) != 0 { + t.Fatalf("summary: %+v", sum) + } + ms, total, _ := s.ListMessages(l.ID, 10, 0) + if total != 3 || ms[0].Subject != "Lunch" { + t.Fatalf("messages: %+v", ms) + } + var reply store.Message + for _, m := range ms { + if m.Subject == "Re: Kickoff" { + reply, _ = s.GetMessage(m.ID) + } + } + if reply.Sender != "Bob" || reply.Body != "agreed\n" || !reply.Date.Equal(time.Date(2024, 1, 1, 11, 0, 0, 0, time.UTC)) { + t.Fatalf("reply: %+v", reply) + } + root, _ := s.GetMessage(reply.ThreadID) + if root.Subject != "Kickoff" { + t.Fatalf("the reply must be threaded under its parent, got %q", root.Subject) + } +} + +func TestImportIsIdempotent(t *testing.T) { + s, l := newList(t) + mbox := mboxMsg("A ", "One", "<1@x>", "", "", "b1") + mboxMsg("B ", "Two", "", "", "", "b2") + if sum, _ := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow); sum.Imported != 2 { + t.Fatalf("first: %+v", sum) + } + sum, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow) + if err != nil || sum.Imported != 0 || sum.Duplicates != 2 || sum.Failed != 0 { + t.Fatalf("second import must add nothing, even for a message without a Message-ID: %+v %v", sum, err) + } + if _, total, _ := s.ListMessages(l.ID, 10, 0); total != 2 { + t.Fatalf("total = %d", total) + } +} + +func TestImportRebuildsThreadsWhateverTheOrder(t *testing.T) { + s, l := newList(t) + mbox := mboxMsg("C ", "Re: Re: T", "", "", "", "c") + + mboxMsg("B ", "Re: T", "", "", "", "b") + + mboxMsg("A ", "T", "", "", "", "a") + if _, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow); err != nil { + t.Fatal(err) + } + ms, _, _ := s.ListMessages(l.ID, 10, 0) + threads := map[int64]int{} + for _, m := range ms { + threads[m.ThreadID]++ + } + if len(threads) != 1 { + t.Fatalf("one thread expected, got %v", threads) + } +} + +func TestImportReportsBadMessagesAndKeepsGoing(t *testing.T) { + s, l := newList(t) + mbox := mboxMsg("A ", "Good one", "<1@x>", "", "", "ok") + + "From x@y.z Tue Jan 2 15:04:05 2024\n\n\nthis message has no headers at all\n\n" + + mboxMsg("B ", "Good two", "<2@x>", "", "", "ok") + sum, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow) + if err != nil { + t.Fatal(err) + } + if sum.Imported != 2 || sum.Failed != 1 || len(sum.Errors) != 1 || !strings.Contains(sum.Errors[0], "message 2") { + t.Fatalf("summary: %+v", sum) + } +} + +func TestImportSkipsOversizedMessages(t *testing.T) { + s, l := newList(t) + big := strings.Repeat("0123456789", MaxImportMessage/10+10) + mbox := mboxMsg("A ", "Small", "<1@x>", "", "", "ok") + mboxMsg("B ", "Huge", "<2@x>", "", "", big) + + mboxMsg("C ", "Small too", "<3@x>", "", "", "ok") + sum, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow) + if err != nil { + t.Fatal(err) + } + if sum.Imported != 2 || sum.Failed != 1 || !strings.Contains(sum.Errors[0], "message 2") || !strings.Contains(sum.Errors[0], "too large") { + t.Fatalf("summary: %+v", sum) + } +} + +func TestImportDateFallbacks(t *testing.T) { + s, l := newList(t) + mbox := mboxMsg("A ", "has Date", "<1@x>", "", "Mon, 01 Jan 2024 10:00:00 +0000", "b") + + mboxMsg("B ", "from envelope", "<2@x>", "", "", "b") + + "From c@x.com Fri Feb 23 08:30 2024\nFrom: C \nSubject: nothing\nMessage-ID: <3@x>\n\nb\n\n" + if _, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow); err != nil { + t.Fatal(err) + } + ms, _, _ := s.ListMessages(l.ID, 10, 0) + dates := map[string]time.Time{} + for _, m := range ms { + dates[m.Subject] = m.Date + } + if !dates["has Date"].Equal(time.Date(2024, 1, 1, 10, 0, 0, 0, time.UTC)) { + t.Errorf("Date header wins: %v", dates["has Date"]) + } + if !dates["from envelope"].Equal(time.Date(2024, 1, 2, 15, 4, 5, 0, time.UTC)) { + t.Errorf("the From line is the fallback: %v", dates["from envelope"]) + } + if !dates["nothing"].Equal(time.Date(2024, 2, 23, 8, 30, 0, 0, time.UTC)) { + t.Errorf("minute-precision envelope date: %v", dates["nothing"]) + } +} + +func TestImportDateFallsBackToNowAsALastResort(t *testing.T) { + s, l := newList(t) + // A valid separator whose date has trailing junk, so it cannot be parsed, and no Date header. + mbox := "From a@x.com Tue Jan 2 15:04:05 2024 extra junk after\nFrom: A \nSubject: undated\nMessage-ID: <1@x>\n\nb\n" + sum, _ := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow) + if sum.Imported != 1 { + t.Fatalf("%+v", sum) + } + ms, _, _ := s.ListMessages(l.ID, 10, 0) + if !ms[0].Date.Equal(importNow) { + t.Fatalf("date = %v, want the import time", ms[0].Date) + } +} + +func TestImportCapsTheErrorListButCountsEverything(t *testing.T) { + s, l := newList(t) + var b strings.Builder + for i := 0; i < 30; i++ { + b.WriteString("From x@y.z Tue Jan 2 15:04:05 2024\n\nno headers here\n\n") + } + sum, err := ImportMbox(s, l.ID, strings.NewReader(b.String()), importNow) + if err != nil { + t.Fatal(err) + } + if sum.Failed != 30 || len(sum.Errors) != MaxReportedErrors { + t.Fatalf("failed=%d errors=%d", sum.Failed, len(sum.Errors)) + } +} + +func TestImportWorksWhateverTheArchivePolicy(t *testing.T) { + s, l := newList(t) + s.SetPolicies(l.ID, store.PostMembers, store.JoinOpen, store.ArchiveOff) + sum, _ := ImportMbox(s, l.ID, strings.NewReader(mboxMsg("A ", "S", "<1@x>", "", "", "b")), importNow) + if sum.Imported != 1 { + t.Fatalf("an explicit import is not governed by the live-mail policy: %+v", sum) + } +} + +func TestImportLargeFileCrossesBatchBoundaries(t *testing.T) { + s, l := newList(t) + var b strings.Builder + n := importBatch*2 + 50 + for i := 0; i < n; i++ { + parent := "" + if i > 0 && i%7 != 0 { + parent = fmt.Sprintf("", i-1) + } + b.WriteString(mboxMsg("A ", fmt.Sprintf("Subject %d", i), fmt.Sprintf("", i), parent, "", "body")) + } + sum, err := ImportMbox(s, l.ID, strings.NewReader(b.String()), importNow) + if err != nil || sum.Imported != n || sum.Failed != 0 { + t.Fatalf("summary: %+v %v", sum, err) + } + if _, total, _ := s.ListMessages(l.ID, 1, 0); total != n { + t.Fatalf("stored %d of %d", total, n) + } +} + +func TestImportDoesNotTouchOtherLists(t *testing.T) { + s, l := newList(t) + other, _ := s.CreateList("ops", "example.org", "Ops") + ImportMbox(s, l.ID, strings.NewReader(mboxMsg("A ", "S", "<1@x>", "", "", "b")), importNow) + if _, total, _ := s.ListMessages(other.ID, 10, 0); total != 0 { + t.Fatalf("other list has %d messages", total) + } +} + +type brokenReader struct { + data string + done bool +} + +func (b *brokenReader) Read(p []byte) (int, error) { + if !b.done { + b.done = true + return copy(p, b.data), nil + } + return 0, errors.New("connection reset") +} + +func TestImportReturnsTheProgressMadeBeforeAReadError(t *testing.T) { + s, l := newList(t) + data := mboxMsg("A ", "First", "<1@x>", "", "", "b") + mboxMsg("B ", "Second", "<2@x>", "", "", "b") + sum, err := ImportMbox(s, l.ID, &brokenReader{data: data}, importNow) + if err == nil || !strings.Contains(err.Error(), "connection reset") { + t.Fatalf("want the read error, got %v", err) + } + if sum.Imported != 1 { + t.Fatalf("the messages read before the failure are kept: %+v", sum) + } + if _, total, _ := s.ListMessages(l.ID, 10, 0); total != 1 { + t.Fatalf("stored %d", total) + } +}