package archive import ( "errors" "fmt" "strings" "testing" "time" "git.kmalloc.org/squid/parsimony/internal/store" ) var importNow = time.Date(2031, 5, 6, 7, 8, 9, 0, time.UTC) func newList(t *testing.T) (*store.Store, store.List) { t.Helper() s, err := store.Open(":memory:") if err != nil { t.Fatal(err) } t.Cleanup(func() { s.Close() }) l, _ := s.CreateList("dev", "example.org", "Dev") return s, l } func mboxMsg(from, subject, id, replyTo, date, body string) string { addr := from if i := strings.Index(from, "<"); i >= 0 { addr = strings.Trim(from[i:], "<>") } h := "From " + addr + " Tue Jan 2 15:04:05 2024\n" h += "From: " + from + "\nSubject: " + subject + "\n" if id != "" { h += "Message-ID: " + id + "\n" } if replyTo != "" { h += "In-Reply-To: " + replyTo + "\n" } if date != "" { h += "Date: " + date + "\n" } return h + "\n" + body + "\n\n" } func TestImportMbox(t *testing.T) { s, l := newList(t) mbox := mboxMsg("Alice ", "Kickoff", "<1@x>", "", "Mon, 01 Jan 2024 10:00:00 +0000", "let's start") + mboxMsg("Bob ", "Re: Kickoff", "<2@x>", "<1@x>", "Mon, 01 Jan 2024 11:00:00 +0000", "agreed") + mboxMsg("Carol ", "Lunch", "<3@x>", "", "Tue, 02 Jan 2024 12:00:00 +0000", "pizza?") sum, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow) if err != nil { t.Fatal(err) } if sum.Imported != 3 || sum.Duplicates != 0 || sum.Failed != 0 || len(sum.Errors) != 0 { t.Fatalf("summary: %+v", sum) } ms, total, _ := s.ListMessages(l.ID, 10, 0) if total != 3 || ms[0].Subject != "Lunch" { t.Fatalf("messages: %+v", ms) } var reply store.Message for _, m := range ms { if m.Subject == "Re: Kickoff" { reply, _ = s.GetMessage(m.ID) } } if reply.Sender != "Bob" || reply.Body != "agreed\n" || !reply.Date.Equal(time.Date(2024, 1, 1, 11, 0, 0, 0, time.UTC)) { t.Fatalf("reply: %+v", reply) } root, _ := s.GetMessage(reply.ThreadID) if root.Subject != "Kickoff" { t.Fatalf("the reply must be threaded under its parent, got %q", root.Subject) } } func TestImportIsIdempotent(t *testing.T) { s, l := newList(t) mbox := mboxMsg("A ", "One", "<1@x>", "", "", "b1") + mboxMsg("B ", "Two", "", "", "", "b2") if sum, _ := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow); sum.Imported != 2 { t.Fatalf("first: %+v", sum) } sum, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow) if err != nil || sum.Imported != 0 || sum.Duplicates != 2 || sum.Failed != 0 { t.Fatalf("second import must add nothing, even for a message without a Message-ID: %+v %v", sum, err) } if _, total, _ := s.ListMessages(l.ID, 10, 0); total != 2 { t.Fatalf("total = %d", total) } } func TestImportRebuildsThreadsWhateverTheOrder(t *testing.T) { s, l := newList(t) mbox := mboxMsg("C ", "Re: Re: T", "", "", "", "c") + mboxMsg("B ", "Re: T", "", "", "", "b") + mboxMsg("A ", "T", "", "", "", "a") if _, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow); err != nil { t.Fatal(err) } ms, _, _ := s.ListMessages(l.ID, 10, 0) threads := map[int64]int{} for _, m := range ms { threads[m.ThreadID]++ } if len(threads) != 1 { t.Fatalf("one thread expected, got %v", threads) } } func TestImportReportsBadMessagesAndKeepsGoing(t *testing.T) { s, l := newList(t) mbox := mboxMsg("A ", "Good one", "<1@x>", "", "", "ok") + "From x@y.z Tue Jan 2 15:04:05 2024\n\n\nthis message has no headers at all\n\n" + mboxMsg("B ", "Good two", "<2@x>", "", "", "ok") sum, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow) if err != nil { t.Fatal(err) } if sum.Imported != 2 || sum.Failed != 1 || len(sum.Errors) != 1 || !strings.Contains(sum.Errors[0], "message 2") { t.Fatalf("summary: %+v", sum) } } func TestImportSkipsOversizedMessages(t *testing.T) { s, l := newList(t) big := strings.Repeat("0123456789", MaxImportMessage/10+10) mbox := mboxMsg("A ", "Small", "<1@x>", "", "", "ok") + mboxMsg("B ", "Huge", "<2@x>", "", "", big) + mboxMsg("C ", "Small too", "<3@x>", "", "", "ok") sum, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow) if err != nil { t.Fatal(err) } if sum.Imported != 2 || sum.Failed != 1 || !strings.Contains(sum.Errors[0], "message 2") || !strings.Contains(sum.Errors[0], "too large") { t.Fatalf("summary: %+v", sum) } } func TestImportDateFallbacks(t *testing.T) { s, l := newList(t) mbox := mboxMsg("A ", "has Date", "<1@x>", "", "Mon, 01 Jan 2024 10:00:00 +0000", "b") + mboxMsg("B ", "from envelope", "<2@x>", "", "", "b") + "From c@x.com Fri Feb 23 08:30 2024\nFrom: C \nSubject: nothing\nMessage-ID: <3@x>\n\nb\n\n" if _, err := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow); err != nil { t.Fatal(err) } ms, _, _ := s.ListMessages(l.ID, 10, 0) dates := map[string]time.Time{} for _, m := range ms { dates[m.Subject] = m.Date } if !dates["has Date"].Equal(time.Date(2024, 1, 1, 10, 0, 0, 0, time.UTC)) { t.Errorf("Date header wins: %v", dates["has Date"]) } if !dates["from envelope"].Equal(time.Date(2024, 1, 2, 15, 4, 5, 0, time.UTC)) { t.Errorf("the From line is the fallback: %v", dates["from envelope"]) } if !dates["nothing"].Equal(time.Date(2024, 2, 23, 8, 30, 0, 0, time.UTC)) { t.Errorf("minute-precision envelope date: %v", dates["nothing"]) } } func TestImportDateFallsBackToNowAsALastResort(t *testing.T) { s, l := newList(t) // A valid separator whose date has trailing junk, so it cannot be parsed, and no Date header. mbox := "From a@x.com Tue Jan 2 15:04:05 2024 extra junk after\nFrom: A \nSubject: undated\nMessage-ID: <1@x>\n\nb\n" sum, _ := ImportMbox(s, l.ID, strings.NewReader(mbox), importNow) if sum.Imported != 1 { t.Fatalf("%+v", sum) } ms, _, _ := s.ListMessages(l.ID, 10, 0) if !ms[0].Date.Equal(importNow) { t.Fatalf("date = %v, want the import time", ms[0].Date) } } func TestImportCapsTheErrorListButCountsEverything(t *testing.T) { s, l := newList(t) var b strings.Builder for i := 0; i < 30; i++ { b.WriteString("From x@y.z Tue Jan 2 15:04:05 2024\n\nno headers here\n\n") } sum, err := ImportMbox(s, l.ID, strings.NewReader(b.String()), importNow) if err != nil { t.Fatal(err) } if sum.Failed != 30 || len(sum.Errors) != MaxReportedErrors { t.Fatalf("failed=%d errors=%d", sum.Failed, len(sum.Errors)) } } func TestImportWorksWhateverTheArchivePolicy(t *testing.T) { s, l := newList(t) s.SetPolicies(l.ID, store.PostMembers, store.JoinOpen, store.ArchiveOff) sum, _ := ImportMbox(s, l.ID, strings.NewReader(mboxMsg("A ", "S", "<1@x>", "", "", "b")), importNow) if sum.Imported != 1 { t.Fatalf("an explicit import is not governed by the live-mail policy: %+v", sum) } } func TestImportLargeFileCrossesBatchBoundaries(t *testing.T) { s, l := newList(t) var b strings.Builder n := importBatch*2 + 50 for i := 0; i < n; i++ { parent := "" if i > 0 && i%7 != 0 { parent = fmt.Sprintf("", i-1) } b.WriteString(mboxMsg("A ", fmt.Sprintf("Subject %d", i), fmt.Sprintf("", i), parent, "", "body")) } sum, err := ImportMbox(s, l.ID, strings.NewReader(b.String()), importNow) if err != nil || sum.Imported != n || sum.Failed != 0 { t.Fatalf("summary: %+v %v", sum, err) } if _, total, _ := s.ListMessages(l.ID, 1, 0); total != n { t.Fatalf("stored %d of %d", total, n) } } func TestImportDoesNotTouchOtherLists(t *testing.T) { s, l := newList(t) other, _ := s.CreateList("ops", "example.org", "Ops") ImportMbox(s, l.ID, strings.NewReader(mboxMsg("A ", "S", "<1@x>", "", "", "b")), importNow) if _, total, _ := s.ListMessages(other.ID, 10, 0); total != 0 { t.Fatalf("other list has %d messages", total) } } type brokenReader struct { data string done bool } func (b *brokenReader) Read(p []byte) (int, error) { if !b.done { b.done = true return copy(p, b.data), nil } return 0, errors.New("connection reset") } func TestImportReturnsTheProgressMadeBeforeAReadError(t *testing.T) { s, l := newList(t) data := mboxMsg("A ", "First", "<1@x>", "", "", "b") + mboxMsg("B ", "Second", "<2@x>", "", "", "b") sum, err := ImportMbox(s, l.ID, &brokenReader{data: data}, importNow) if err == nil || !strings.Contains(err.Error(), "connection reset") { t.Fatalf("want the read error, got %v", err) } if sum.Imported != 1 { t.Fatalf("the messages read before the failure are kept: %+v", sum) } if _, total, _ := s.ListMessages(l.ID, 10, 0); total != 1 { t.Fatalf("stored %d", total) } }