feat(archive): import messages from an mbox
- Add ImportMbox: batches of 200, idempotent by Message-ID, threads rebuilt at the end - Date fallbacks: Date header, then the From line, then the import time - Report failed and oversized messages without stopping, keep progress on a read error - Import regardless of the list's archive policy
This commit is contained in:
parent
ba6c37bc21
commit
45a61ef294
2 changed files with 360 additions and 0 deletions
106
internal/archive/import.go
Normal file
106
internal/archive/import.go
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
package archive
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"codeberg.org/squidd/parsimony/internal/mbox"
|
||||
"codeberg.org/squidd/parsimony/internal/store"
|
||||
)
|
||||
|
||||
const (
|
||||
// MaxImportMessage is the largest message accepted from an mbox, in bytes.
|
||||
MaxImportMessage = 25 << 20
|
||||
// MaxReportedErrors caps the error messages kept in a summary; Failed still counts every one.
|
||||
MaxReportedErrors = 20
|
||||
|
||||
importBatch = 200 // messages per transaction
|
||||
)
|
||||
|
||||
// ImportSummary reports what an import did.
|
||||
type ImportSummary struct {
|
||||
Imported int `json:"imported"` // new messages stored
|
||||
Duplicates int `json:"duplicates"` // already in the archive (same Message-ID)
|
||||
Failed int `json:"failed"` // unreadable or too large
|
||||
Errors []string `json:"errors"` // the first MaxReportedErrors failures
|
||||
}
|
||||
|
||||
// ImportMbox archives every message of an mbox into a list. It is idempotent
|
||||
// (a message is identified by its Message-ID) and works whatever the list's
|
||||
// archive policy is: the policy only governs live mail. Threads are rebuilt at
|
||||
// the end, so the order of the messages in the file does not matter. now is the
|
||||
// date of last resort for a message with neither a Date header nor a dated
|
||||
// "From " line. On a read error the messages read so far are kept and returned
|
||||
// in the summary together with the error.
|
||||
func ImportMbox(st *store.Store, listID int64, r io.Reader, now time.Time) (ImportSummary, error) {
|
||||
sum := ImportSummary{Errors: []string{}}
|
||||
var batch []store.Message
|
||||
|
||||
fail := func(index int, err error) {
|
||||
sum.Failed++
|
||||
if len(sum.Errors) < MaxReportedErrors {
|
||||
sum.Errors = append(sum.Errors, fmt.Sprintf("message %d: %v", index, err))
|
||||
}
|
||||
}
|
||||
flush := func() error {
|
||||
results, err := st.ArchiveBatch(listID, batch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, res := range results {
|
||||
if res.Created {
|
||||
sum.Imported++
|
||||
} else {
|
||||
sum.Duplicates++
|
||||
}
|
||||
}
|
||||
batch = batch[:0]
|
||||
return nil
|
||||
}
|
||||
finish := func(readErr error) (ImportSummary, error) {
|
||||
if err := flush(); err != nil {
|
||||
return sum, err
|
||||
}
|
||||
if sum.Imported > 0 {
|
||||
if _, err := st.Rethread(listID); err != nil {
|
||||
return sum, err
|
||||
}
|
||||
}
|
||||
return sum, readErr
|
||||
}
|
||||
|
||||
rd := mbox.NewReader(r, MaxImportMessage)
|
||||
for {
|
||||
m, err := rd.Next()
|
||||
if err == io.EOF {
|
||||
return finish(nil)
|
||||
}
|
||||
if errors.Is(err, mbox.ErrTooLarge) {
|
||||
fail(m.Index, fmt.Errorf("too large (limit %d MiB)", MaxImportMessage>>20))
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return finish(fmt.Errorf("read mbox: %w", err))
|
||||
}
|
||||
fallback := now
|
||||
if d, ok := m.Date(); ok {
|
||||
fallback = d
|
||||
}
|
||||
en, err := Parse(m.Data, fallback)
|
||||
if err != nil {
|
||||
fail(m.Index, err)
|
||||
continue
|
||||
}
|
||||
batch = append(batch, store.Message{
|
||||
MessageID: en.MessageID, InReplyTo: en.InReplyTo, Subject: en.Subject,
|
||||
Sender: en.Sender, Date: en.Date, Body: en.Body,
|
||||
})
|
||||
if len(batch) >= importBatch {
|
||||
if err := flush(); err != nil {
|
||||
return sum, err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
254
internal/archive/import_test.go
Normal file
254
internal/archive/import_test.go
Normal file
|
|
@ -0,0 +1,254 @@
|
|||
package archive
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"codeberg.org/squidd/parsimony/internal/store"
|
||||
)
|
||||
|
||||
var importNow = time.Date(2031, 5, 6, 7, 8, 9, 0, time.UTC)
|
||||
|
||||
func newList(t *testing.T) (*store.Store, store.List) {
|
||||
t.Helper()
|
||||
s, err := store.Open(":memory:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { s.Close() })
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
return s, l
|
||||
}
|
||||
|
||||
func mboxMsg(from, subject, id, replyTo, date, body string) string {
|
||||
addr := from
|
||||
if i := strings.Index(from, "<"); i >= 0 {
|
||||
addr = strings.Trim(from[i:], "<>")
|
||||
}
|
||||
h := "From " + addr + " Tue Jan 2 15:04:05 2024\n"
|
||||
h += "From: " + from + "\nSubject: " + subject + "\n"
|
||||
if id != "" {
|
||||
h += "Message-ID: " + id + "\n"
|
||||
}
|
||||
if replyTo != "" {
|
||||
h += "In-Reply-To: " + replyTo + "\n"
|
||||
}
|
||||
if date != "" {
|
||||
h += "Date: " + date + "\n"
|
||||
}
|
||||
return h + "\n" + body + "\n\n"
|
||||
}
|
||||
|
||||
func TestImportMbox(t *testing.T) {
|
||||
s, l := newList(t)
|
||||
mbox := mboxMsg("Alice <alice@example.com>", "Kickoff", "<1@x>", "", "Mon, 01 Jan 2024 10:00:00 +0000", "let's start") +
|
||||
mboxMsg("Bob <bob@example.com>", "Re: Kickoff", "<2@x>", "<1@x>", "Mon, 01 Jan 2024 11:00:00 +0000", "agreed") +
|
||||
mboxMsg("Carol <carol@example.com>", "Lunch", "<3@x>", "", "Tue, 02 Jan 2024 12:00:00 +0000", "pizza?")
|
||||
sum, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sum.Imported != 3 || sum.Duplicates != 0 || sum.Failed != 0 || len(sum.Errors) != 0 {
|
||||
t.Fatalf("summary: %+v", sum)
|
||||
}
|
||||
ms, total, _ := s.ListMessages(l.ID, 10, 0)
|
||||
if total != 3 || ms[0].Subject != "Lunch" {
|
||||
t.Fatalf("messages: %+v", ms)
|
||||
}
|
||||
var reply store.Message
|
||||
for _, m := range ms {
|
||||
if m.Subject == "Re: Kickoff" {
|
||||
reply, _ = s.GetMessage(m.ID)
|
||||
}
|
||||
}
|
||||
if reply.Sender != "Bob" || reply.Body != "agreed\n" || !reply.Date.Equal(time.Date(2024, 1, 1, 11, 0, 0, 0, time.UTC)) {
|
||||
t.Fatalf("reply: %+v", reply)
|
||||
}
|
||||
root, _ := s.GetMessage(reply.ThreadID)
|
||||
if root.Subject != "Kickoff" {
|
||||
t.Fatalf("the reply must be threaded under its parent, got %q", root.Subject)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportIsIdempotent(t *testing.T) {
|
||||
s, l := newList(t)
|
||||
mbox := mboxMsg("A <a@x.com>", "One", "<1@x>", "", "", "b1") + mboxMsg("B <b@x.com>", "Two", "", "", "", "b2")
|
||||
if sum, _ := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow); sum.Imported != 2 {
|
||||
t.Fatalf("first: %+v", sum)
|
||||
}
|
||||
sum, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow)
|
||||
if err != nil || sum.Imported != 0 || sum.Duplicates != 2 || sum.Failed != 0 {
|
||||
t.Fatalf("second import must add nothing, even for a message without a Message-ID: %+v %v", sum, err)
|
||||
}
|
||||
if _, total, _ := s.ListMessages(l.ID, 10, 0); total != 2 {
|
||||
t.Fatalf("total = %d", total)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportRebuildsThreadsWhateverTheOrder(t *testing.T) {
|
||||
s, l := newList(t)
|
||||
mbox := mboxMsg("C <c@x.com>", "Re: Re: T", "<c@x>", "<b@x>", "", "c") +
|
||||
mboxMsg("B <b@x.com>", "Re: T", "<b@x>", "<a@x>", "", "b") +
|
||||
mboxMsg("A <a@x.com>", "T", "<a@x>", "", "", "a")
|
||||
if _, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ms, _, _ := s.ListMessages(l.ID, 10, 0)
|
||||
threads := map[int64]int{}
|
||||
for _, m := range ms {
|
||||
threads[m.ThreadID]++
|
||||
}
|
||||
if len(threads) != 1 {
|
||||
t.Fatalf("one thread expected, got %v", threads)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportReportsBadMessagesAndKeepsGoing(t *testing.T) {
|
||||
s, l := newList(t)
|
||||
mbox := mboxMsg("A <a@x.com>", "Good one", "<1@x>", "", "", "ok") +
|
||||
"From x@y.z Tue Jan 2 15:04:05 2024\n\n\nthis message has no headers at all\n\n" +
|
||||
mboxMsg("B <b@x.com>", "Good two", "<2@x>", "", "", "ok")
|
||||
sum, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sum.Imported != 2 || sum.Failed != 1 || len(sum.Errors) != 1 || !strings.Contains(sum.Errors[0], "message 2") {
|
||||
t.Fatalf("summary: %+v", sum)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportSkipsOversizedMessages(t *testing.T) {
|
||||
s, l := newList(t)
|
||||
big := strings.Repeat("0123456789", MaxImportMessage/10+10)
|
||||
mbox := mboxMsg("A <a@x.com>", "Small", "<1@x>", "", "", "ok") + mboxMsg("B <b@x.com>", "Huge", "<2@x>", "", "", big) +
|
||||
mboxMsg("C <c@x.com>", "Small too", "<3@x>", "", "", "ok")
|
||||
sum, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sum.Imported != 2 || sum.Failed != 1 || !strings.Contains(sum.Errors[0], "message 2") || !strings.Contains(sum.Errors[0], "too large") {
|
||||
t.Fatalf("summary: %+v", sum)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportDateFallbacks(t *testing.T) {
|
||||
s, l := newList(t)
|
||||
mbox := mboxMsg("A <a@x.com>", "has Date", "<1@x>", "", "Mon, 01 Jan 2024 10:00:00 +0000", "b") +
|
||||
mboxMsg("B <b@x.com>", "from envelope", "<2@x>", "", "", "b") +
|
||||
"From c@x.com Fri Feb 23 08:30 2024\nFrom: C <c@x.com>\nSubject: nothing\nMessage-ID: <3@x>\n\nb\n\n"
|
||||
if _, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ms, _, _ := s.ListMessages(l.ID, 10, 0)
|
||||
dates := map[string]time.Time{}
|
||||
for _, m := range ms {
|
||||
dates[m.Subject] = m.Date
|
||||
}
|
||||
if !dates["has Date"].Equal(time.Date(2024, 1, 1, 10, 0, 0, 0, time.UTC)) {
|
||||
t.Errorf("Date header wins: %v", dates["has Date"])
|
||||
}
|
||||
if !dates["from envelope"].Equal(time.Date(2024, 1, 2, 15, 4, 5, 0, time.UTC)) {
|
||||
t.Errorf("the From line is the fallback: %v", dates["from envelope"])
|
||||
}
|
||||
if !dates["nothing"].Equal(time.Date(2024, 2, 23, 8, 30, 0, 0, time.UTC)) {
|
||||
t.Errorf("minute-precision envelope date: %v", dates["nothing"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportDateFallsBackToNowAsALastResort(t *testing.T) {
|
||||
s, l := newList(t)
|
||||
// A valid separator whose date has trailing junk, so it cannot be parsed, and no Date header.
|
||||
mbox := "From a@x.com Tue Jan 2 15:04:05 2024 extra junk after\nFrom: A <a@x.com>\nSubject: undated\nMessage-ID: <1@x>\n\nb\n"
|
||||
sum, _ := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow)
|
||||
if sum.Imported != 1 {
|
||||
t.Fatalf("%+v", sum)
|
||||
}
|
||||
ms, _, _ := s.ListMessages(l.ID, 10, 0)
|
||||
if !ms[0].Date.Equal(importNow) {
|
||||
t.Fatalf("date = %v, want the import time", ms[0].Date)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportCapsTheErrorListButCountsEverything(t *testing.T) {
|
||||
s, l := newList(t)
|
||||
var b strings.Builder
|
||||
for i := 0; i < 30; i++ {
|
||||
b.WriteString("From x@y.z Tue Jan 2 15:04:05 2024\n\nno headers here\n\n")
|
||||
}
|
||||
sum, err := ImportMbox(s, l.ID, strings.NewReader(b.String()), importNow)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sum.Failed != 30 || len(sum.Errors) != MaxReportedErrors {
|
||||
t.Fatalf("failed=%d errors=%d", sum.Failed, len(sum.Errors))
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportWorksWhateverTheArchivePolicy(t *testing.T) {
|
||||
s, l := newList(t)
|
||||
s.SetPolicies(l.ID, store.PostMembers, store.JoinOpen, store.ArchiveOff)
|
||||
sum, _ := ImportMbox(s, l.ID, strings.NewReader(mboxMsg("A <a@x.com>", "S", "<1@x>", "", "", "b")), importNow)
|
||||
if sum.Imported != 1 {
|
||||
t.Fatalf("an explicit import is not governed by the live-mail policy: %+v", sum)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportLargeFileCrossesBatchBoundaries(t *testing.T) {
|
||||
s, l := newList(t)
|
||||
var b strings.Builder
|
||||
n := importBatch*2 + 50
|
||||
for i := 0; i < n; i++ {
|
||||
parent := ""
|
||||
if i > 0 && i%7 != 0 {
|
||||
parent = fmt.Sprintf("<m%d@x>", i-1)
|
||||
}
|
||||
b.WriteString(mboxMsg("A <a@x.com>", fmt.Sprintf("Subject %d", i), fmt.Sprintf("<m%d@x>", i), parent, "", "body"))
|
||||
}
|
||||
sum, err := ImportMbox(s, l.ID, strings.NewReader(b.String()), importNow)
|
||||
if err != nil || sum.Imported != n || sum.Failed != 0 {
|
||||
t.Fatalf("summary: %+v %v", sum, err)
|
||||
}
|
||||
if _, total, _ := s.ListMessages(l.ID, 1, 0); total != n {
|
||||
t.Fatalf("stored %d of %d", total, n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportDoesNotTouchOtherLists(t *testing.T) {
|
||||
s, l := newList(t)
|
||||
other, _ := s.CreateList("ops", "example.org", "Ops")
|
||||
ImportMbox(s, l.ID, strings.NewReader(mboxMsg("A <a@x.com>", "S", "<1@x>", "", "", "b")), importNow)
|
||||
if _, total, _ := s.ListMessages(other.ID, 10, 0); total != 0 {
|
||||
t.Fatalf("other list has %d messages", total)
|
||||
}
|
||||
}
|
||||
|
||||
type brokenReader struct {
|
||||
data string
|
||||
done bool
|
||||
}
|
||||
|
||||
func (b *brokenReader) Read(p []byte) (int, error) {
|
||||
if !b.done {
|
||||
b.done = true
|
||||
return copy(p, b.data), nil
|
||||
}
|
||||
return 0, errors.New("connection reset")
|
||||
}
|
||||
|
||||
func TestImportReturnsTheProgressMadeBeforeAReadError(t *testing.T) {
|
||||
s, l := newList(t)
|
||||
data := mboxMsg("A <a@x.com>", "First", "<1@x>", "", "", "b") + mboxMsg("B <b@x.com>", "Second", "<2@x>", "", "", "b")
|
||||
sum, err := ImportMbox(s, l.ID, &brokenReader{data: data}, importNow)
|
||||
if err == nil || !strings.Contains(err.Error(), "connection reset") {
|
||||
t.Fatalf("want the read error, got %v", err)
|
||||
}
|
||||
if sum.Imported != 1 {
|
||||
t.Fatalf("the messages read before the failure are kept: %+v", sum)
|
||||
}
|
||||
if _, total, _ := s.ListMessages(l.ID, 10, 0); total != 1 {
|
||||
t.Fatalf("stored %d", total)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue