feat(mbox,store): mbox reader, batch archiving and thread rebuilding
- Add internal/mbox: streaming reader with strict From_ detection, mboxrd unescaping, size cap and envelope dates - Add Store.ArchiveBatch: many messages in one transaction - Add Store.Rethread: rebuild threads from In-Reply-To whatever the insertion order, tolerating cycles
This commit is contained in:
parent
3b7e8c0d9a
commit
ba6c37bc21
4 changed files with 599 additions and 4 deletions
147
internal/mbox/mbox.go
Normal file
147
internal/mbox/mbox.go
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
// Package mbox reads mbox archives, such as the ones Mailman exports.
|
||||
//
|
||||
// A message starts at a line like "From alice@example.com Tue Jan 2 15:04:05 2024".
|
||||
// A line that merely begins with "From " in prose does not split a message. Lines
|
||||
// escaped as ">From " are restored (mboxrd); the blank line before a separator
|
||||
// belongs to the separator, not to the message.
|
||||
package mbox
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrTooLarge is returned with a message that exceeded the size limit and was skipped.
|
||||
var ErrTooLarge = errors.New("message too large")
|
||||
|
||||
var (
|
||||
separator = regexp.MustCompile(`^From \S+\s+(Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2}\s+\d{1,2}:\d{2}`)
|
||||
escaped = regexp.MustCompile(`^>+From `)
|
||||
)
|
||||
|
||||
// Message is one message of the mbox.
|
||||
type Message struct {
|
||||
Index int // 1-based position in the file
|
||||
Envelope string // the "From " line
|
||||
Data []byte // the RFC 5322 message; nil when Err is set
|
||||
Err error
|
||||
}
|
||||
|
||||
// Date returns the date of the "From " line, when it has a parsable one.
|
||||
func (m Message) Date() (time.Time, bool) {
|
||||
f := strings.Fields(m.Envelope)
|
||||
if len(f) < 3 || f[0] != "From" {
|
||||
return time.Time{}, false
|
||||
}
|
||||
rest := strings.Join(f[2:], " ")
|
||||
for _, layout := range []string{
|
||||
"Mon Jan 2 15:04:05 2006",
|
||||
"Mon Jan 2 15:04:05 -0700 2006",
|
||||
"Mon Jan 2 15:04:05 MST 2006",
|
||||
"Mon Jan 2 15:04 2006",
|
||||
} {
|
||||
if t, err := time.Parse(layout, rest); err == nil {
|
||||
return t, true
|
||||
}
|
||||
}
|
||||
return time.Time{}, false
|
||||
}
|
||||
|
||||
// Reader returns the messages of an mbox one at a time.
|
||||
type Reader struct {
|
||||
br *bufio.Reader
|
||||
max int
|
||||
next string // envelope of the message that comes next, once seen
|
||||
index int
|
||||
eof bool
|
||||
}
|
||||
|
||||
// NewReader reads from r. Messages larger than maxSize bytes (0 means no limit)
|
||||
// are skipped and reported with ErrTooLarge.
|
||||
func NewReader(r io.Reader, maxSize int) *Reader {
|
||||
return &Reader{br: bufio.NewReaderSize(r, 64*1024), max: maxSize}
|
||||
}
|
||||
|
||||
// readLine returns the next line including its newline. At the end of the input
|
||||
// it returns the last partial line (if any) with io.EOF on the following call.
|
||||
func (r *Reader) readLine() ([]byte, error) {
|
||||
line, err := r.br.ReadBytes('\n')
|
||||
if err == io.EOF && len(line) > 0 {
|
||||
return line, nil
|
||||
}
|
||||
return line, err
|
||||
}
|
||||
|
||||
// Next returns the next message, or io.EOF when there are no more.
|
||||
func (r *Reader) Next() (Message, error) {
|
||||
if r.eof {
|
||||
return Message{}, io.EOF
|
||||
}
|
||||
// Skip anything before the first separator.
|
||||
for r.next == "" {
|
||||
line, err := r.readLine()
|
||||
if err != nil {
|
||||
r.eof = true
|
||||
if err == io.EOF {
|
||||
return Message{}, io.EOF
|
||||
}
|
||||
return Message{}, err
|
||||
}
|
||||
if s := strings.TrimRight(string(line), "\r\n"); separator.MatchString(s) {
|
||||
r.next = s
|
||||
}
|
||||
}
|
||||
r.index++
|
||||
msg := Message{Index: r.index, Envelope: r.next}
|
||||
r.next = ""
|
||||
|
||||
var buf bytes.Buffer
|
||||
oversized := false
|
||||
for {
|
||||
line, err := r.readLine()
|
||||
if err != nil {
|
||||
r.eof = true
|
||||
if err != io.EOF {
|
||||
return Message{}, err
|
||||
}
|
||||
break
|
||||
}
|
||||
if s := strings.TrimRight(string(line), "\r\n"); separator.MatchString(s) {
|
||||
r.next = s
|
||||
break
|
||||
}
|
||||
if oversized {
|
||||
continue
|
||||
}
|
||||
if escaped.Match(line) {
|
||||
line = line[1:]
|
||||
}
|
||||
buf.Write(line)
|
||||
if r.max > 0 && buf.Len() > r.max {
|
||||
oversized = true
|
||||
buf.Reset()
|
||||
}
|
||||
}
|
||||
if oversized {
|
||||
msg.Err = ErrTooLarge
|
||||
return msg, ErrTooLarge
|
||||
}
|
||||
msg.Data = trimSeparatorLine(buf.Bytes())
|
||||
return msg, nil
|
||||
}
|
||||
|
||||
// trimSeparatorLine drops the one blank line that precedes a separator.
|
||||
func trimSeparatorLine(b []byte) []byte {
|
||||
switch {
|
||||
case bytes.HasSuffix(b, []byte("\r\n\r\n")):
|
||||
return b[:len(b)-2]
|
||||
case bytes.HasSuffix(b, []byte("\n\n")):
|
||||
return b[:len(b)-1]
|
||||
}
|
||||
return b
|
||||
}
|
||||
165
internal/mbox/mbox_test.go
Normal file
165
internal/mbox/mbox_test.go
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
package mbox
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func readAll(t *testing.T, input string, maxSize int) []Message {
|
||||
t.Helper()
|
||||
r := NewReader(strings.NewReader(input), maxSize)
|
||||
var out []Message
|
||||
for {
|
||||
m, err := r.Next()
|
||||
if err == io.EOF {
|
||||
return out
|
||||
}
|
||||
if err != nil && !errors.Is(err, ErrTooLarge) {
|
||||
t.Fatalf("next: %v", err)
|
||||
}
|
||||
m.Err = err
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
|
||||
const two = "From alice@example.com Tue Jan 2 15:04:05 2024\n" +
|
||||
"From: alice@example.com\nSubject: one\n\nfirst body\n\n" +
|
||||
"From bob@example.com Wed Jan 3 09:00:00 2024\n" +
|
||||
"From: bob@example.com\nSubject: two\n\nsecond body\n"
|
||||
|
||||
func TestSplitsMessages(t *testing.T) {
|
||||
ms := readAll(t, two, 0)
|
||||
if len(ms) != 2 {
|
||||
t.Fatalf("got %d messages", len(ms))
|
||||
}
|
||||
if string(ms[0].Data) != "From: alice@example.com\nSubject: one\n\nfirst body\n" {
|
||||
t.Fatalf("first = %q", ms[0].Data)
|
||||
}
|
||||
if string(ms[1].Data) != "From: bob@example.com\nSubject: two\n\nsecond body\n" {
|
||||
t.Fatalf("second = %q", ms[1].Data)
|
||||
}
|
||||
if ms[0].Envelope != "From alice@example.com Tue Jan 2 15:04:05 2024" {
|
||||
t.Fatalf("envelope = %q", ms[0].Envelope)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnvelopeDate(t *testing.T) {
|
||||
ms := readAll(t, two, 0)
|
||||
want := time.Date(2024, 1, 2, 15, 4, 5, 0, time.UTC)
|
||||
if got, ok := ms[0].Date(); !ok || !got.Equal(want) {
|
||||
t.Fatalf("date = %v %v", got, ok)
|
||||
}
|
||||
for _, env := range []string{
|
||||
"From x@y.z Tue Jan 2 15:04:05 +0200 2024",
|
||||
"From x@y.z Tue Jan 2 15:04:05 2024",
|
||||
"From x@y.z Tue Jan 2 15:04 2024",
|
||||
"From MAILER-DAEMON Tue Jan 2 15:04:05 2024",
|
||||
} {
|
||||
if _, ok := (Message{Envelope: env}).Date(); !ok {
|
||||
t.Errorf("%q must parse", env)
|
||||
}
|
||||
}
|
||||
for _, env := range []string{"", "From x@y.z", "From x@y.z not a date"} {
|
||||
if _, ok := (Message{Envelope: env}).Date(); ok {
|
||||
t.Errorf("%q must not parse", env)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProseLineStartingWithFromDoesNotSplit(t *testing.T) {
|
||||
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nFrom now on we meet on Fridays.\nFrom: not a separator either\n"
|
||||
ms := readAll(t, in, 0)
|
||||
if len(ms) != 1 || !strings.Contains(string(ms[0].Data), "From now on we meet on Fridays.") {
|
||||
t.Fatalf("messages: %d %q", len(ms), ms[0].Data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMboxrdUnescaping(t *testing.T) {
|
||||
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n>From the log:\n>>From a quoted line\n> From not escaped\nplain\n"
|
||||
ms := readAll(t, in, 0)
|
||||
body := string(ms[0].Data)
|
||||
if !strings.Contains(body, "\nFrom the log:\n") || !strings.Contains(body, "\n>From a quoted line\n") || !strings.Contains(body, "\n> From not escaped\n") {
|
||||
t.Fatalf("body = %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLeadingJunkAndEmptyInput(t *testing.T) {
|
||||
if got := readAll(t, "", 0); len(got) != 0 {
|
||||
t.Fatalf("empty input: %d", len(got))
|
||||
}
|
||||
if got := readAll(t, "\n\nsome preamble\nFrom a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nbody\n", 0); len(got) != 1 {
|
||||
t.Fatalf("preamble must be ignored: %d", len(got))
|
||||
}
|
||||
if got := readAll(t, "just text, no separator at all\n", 0); len(got) != 0 {
|
||||
t.Fatalf("no messages expected: %d", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCRLFAndMissingFinalNewline(t *testing.T) {
|
||||
in := "From a@x.com Tue Jan 2 15:04:05 2024\r\nSubject: s\r\n\r\nbody without newline"
|
||||
ms := readAll(t, in, 0)
|
||||
if len(ms) != 1 || !strings.HasSuffix(string(ms[0].Data), "body without newline") || !strings.Contains(string(ms[0].Data), "Subject: s\r\n") {
|
||||
t.Fatalf("%q", ms[0].Data)
|
||||
}
|
||||
crlf := "From a@x.com Tue Jan 2 15:04:05 2024\r\nSubject: one\r\n\r\nb\r\n\r\nFrom b@x.com Tue Jan 2 15:04:05 2024\r\nSubject: two\r\n\r\nb2\r\n"
|
||||
if got := readAll(t, crlf, 0); len(got) != 2 {
|
||||
t.Fatalf("CRLF separators: %d", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestVeryLongLinesAreFine(t *testing.T) {
|
||||
long := strings.Repeat("x", 300_000)
|
||||
ms := readAll(t, "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\n"+long+"\n", 0)
|
||||
if len(ms) != 1 || !strings.Contains(string(ms[0].Data), long) {
|
||||
t.Fatal("a long line must survive")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOversizedMessagesAreReportedAndSkipped(t *testing.T) {
|
||||
big := strings.Repeat("y\n", 5000)
|
||||
in := "From a@x.com Tue Jan 2 15:04:05 2024\nSubject: ok1\n\nsmall\n" +
|
||||
"From b@x.com Tue Jan 2 15:04:05 2024\nSubject: big\n\n" + big +
|
||||
"From c@x.com Tue Jan 2 15:04:05 2024\nSubject: ok2\n\nsmall\n"
|
||||
ms := readAll(t, in, 1000)
|
||||
if len(ms) != 3 {
|
||||
t.Fatalf("got %d", len(ms))
|
||||
}
|
||||
if ms[0].Err != nil || ms[2].Err != nil {
|
||||
t.Fatalf("small messages must pass: %v %v", ms[0].Err, ms[2].Err)
|
||||
}
|
||||
if !errors.Is(ms[1].Err, ErrTooLarge) || ms[1].Data != nil {
|
||||
t.Fatalf("big message: %v %d bytes", ms[1].Err, len(ms[1].Data))
|
||||
}
|
||||
if ms[1].Envelope != "From b@x.com Tue Jan 2 15:04:05 2024" || !strings.Contains(string(ms[2].Data), "ok2") {
|
||||
t.Fatalf("the reader must resynchronise after a big message: %+v", ms[2])
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexIsOneBased(t *testing.T) {
|
||||
ms := readAll(t, two, 0)
|
||||
if ms[0].Index != 1 || ms[1].Index != 2 {
|
||||
t.Fatalf("indexes: %d %d", ms[0].Index, ms[1].Index)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadErrorsArePropagated(t *testing.T) {
|
||||
r := NewReader(io.MultiReader(strings.NewReader("From a@x.com Tue Jan 2 15:04:05 2024\nSubject: s\n\nbody\n"), errReader{}), 0)
|
||||
var last error
|
||||
for i := 0; i < 5; i++ {
|
||||
_, err := r.Next()
|
||||
if err != nil {
|
||||
last = err
|
||||
break
|
||||
}
|
||||
}
|
||||
if last == nil || last == io.EOF {
|
||||
t.Fatalf("want the read error, got %v", last)
|
||||
}
|
||||
}
|
||||
|
||||
type errReader struct{}
|
||||
|
||||
func (errReader) Read([]byte) (int, error) { return 0, errors.New("disk on fire") }
|
||||
|
|
@ -227,3 +227,159 @@ func TestLongBodyIsStoredAsIs(t *testing.T) {
|
|||
t.Fatalf("body len %d", len(got.Body))
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveBatchInsertsDeduplicatesAndThreads(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
s.ArchiveMessage(l.ID, msg("<old@x>", "Already here", "x"))
|
||||
|
||||
reply := msg("<r@x>", "Re: Topic", "reply")
|
||||
reply.InReplyTo = "<root@x>"
|
||||
res, err := s.ArchiveBatch(l.ID, []Message{
|
||||
msg("<root@x>", "Topic", "root"),
|
||||
reply,
|
||||
msg("<old@x>", "Already here", "x"), // duplicate of an existing message
|
||||
msg("<root@x>", "Topic", "root"), // duplicate inside the batch
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(res) != 4 || !res[0].Created || !res[1].Created || res[2].Created || res[3].Created {
|
||||
t.Fatalf("results: %+v", res)
|
||||
}
|
||||
if res[3].ID != res[0].ID {
|
||||
t.Fatalf("a duplicate reports the id of the stored message: %+v", res)
|
||||
}
|
||||
r, _ := s.GetMessage(res[1].ID)
|
||||
if r.ThreadID != res[0].ID {
|
||||
t.Fatalf("threading inside a batch: thread %d, root %d", r.ThreadID, res[0].ID)
|
||||
}
|
||||
if _, total, _ := s.ListMessages(l.ID, 10, 0); total != 3 {
|
||||
t.Fatalf("total = %d", total)
|
||||
}
|
||||
if got, _, _ := s.SearchMessages(l.ID, "reply", 10, 0); len(got) != 1 {
|
||||
t.Fatal("batch inserts must be searchable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveBatchIsAtomic(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
_, err := s.ArchiveBatch(l.ID, []Message{msg("<a@x>", "A", "a"), {Subject: "no id"}})
|
||||
if err == nil {
|
||||
t.Fatal("a message without an id must fail the batch")
|
||||
}
|
||||
if _, total, _ := s.ListMessages(l.ID, 10, 0); total != 0 {
|
||||
t.Fatalf("nothing may be stored after a failed batch, got %d", total)
|
||||
}
|
||||
if res, err := s.ArchiveBatch(l.ID, nil); err != nil || len(res) != 0 {
|
||||
t.Fatalf("empty batch: %v %v", res, err)
|
||||
}
|
||||
}
|
||||
|
||||
func threadOf(t *testing.T, s *Store, id int64) int64 {
|
||||
t.Helper()
|
||||
m, err := s.GetMessage(id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m.ThreadID
|
||||
}
|
||||
|
||||
func TestRethreadFixesMessagesImportedOutOfOrder(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
// Children first, root last: at insertion time none of the parents exists.
|
||||
c2 := msg("<c2@x>", "Re: Re: T", "c2")
|
||||
c2.InReplyTo = "<c1@x>"
|
||||
c1 := msg("<c1@x>", "Re: T", "c1")
|
||||
c1.InReplyTo = "<root@x>"
|
||||
id2, _, _ := s.ArchiveMessage(l.ID, c2)
|
||||
id1, _, _ := s.ArchiveMessage(l.ID, c1)
|
||||
root, _, _ := s.ArchiveMessage(l.ID, msg("<root@x>", "T", "root"))
|
||||
other, _, _ := s.ArchiveMessage(l.ID, msg("<other@x>", "Other", "o"))
|
||||
if threadOf(t, s, id1) == root {
|
||||
t.Fatal("precondition: the thread is broken before rethreading")
|
||||
}
|
||||
|
||||
changed, err := s.Rethread(l.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if changed != 2 {
|
||||
t.Fatalf("changed = %d, want the 2 broken messages", changed)
|
||||
}
|
||||
for _, id := range []int64{id1, id2, root} {
|
||||
if got := threadOf(t, s, id); got != root {
|
||||
t.Fatalf("message %d thread = %d, want %d", id, got, root)
|
||||
}
|
||||
}
|
||||
if threadOf(t, s, other) != other {
|
||||
t.Fatal("an unrelated message keeps its own thread")
|
||||
}
|
||||
if again, _ := s.Rethread(l.ID); again != 0 {
|
||||
t.Fatalf("rethreading twice changes nothing, got %d", again)
|
||||
}
|
||||
if th, _ := s.ThreadMessages(l.ID, root); len(th) != 3 {
|
||||
t.Fatalf("thread has %d messages", len(th))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRethreadSurvivesCyclesAndSelfReplies(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
l, _ := s.CreateList("dev", "example.org", "Dev")
|
||||
a := msg("<a@x>", "A", "a")
|
||||
a.InReplyTo = "<b@x>"
|
||||
b := msg("<b@x>", "B", "b")
|
||||
b.InReplyTo = "<a@x>"
|
||||
self := msg("<s@x>", "Self", "s")
|
||||
self.InReplyTo = "<s@x>"
|
||||
ia, _, _ := s.ArchiveMessage(l.ID, a)
|
||||
ib, _, _ := s.ArchiveMessage(l.ID, b)
|
||||
is, _, _ := s.ArchiveMessage(l.ID, self)
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { _, err := s.Rethread(l.ID); done <- err }()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("Rethread looped forever on a cycle")
|
||||
}
|
||||
if threadOf(t, s, ia) != threadOf(t, s, ib) {
|
||||
t.Fatal("the two messages of a cycle share a thread")
|
||||
}
|
||||
if got := threadOf(t, s, ia); got != ia {
|
||||
t.Fatalf("the cycle's thread is its smallest id (%d), got %d", ia, got)
|
||||
}
|
||||
if threadOf(t, s, is) != is {
|
||||
t.Fatal("a message that replies to itself is its own thread")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRethreadOnlyTouchesTheGivenList(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
a, _ := s.CreateList("a", "example.org", "A")
|
||||
b, _ := s.CreateList("b", "example.org", "B")
|
||||
child := msg("<c@x>", "Re", "c")
|
||||
child.InReplyTo = "<p@x>"
|
||||
cb, _, _ := s.ArchiveMessage(b.ID, child)
|
||||
s.ArchiveMessage(b.ID, msg("<p@x>", "P", "p"))
|
||||
// A parent with the same Message-ID in another list must not be used.
|
||||
s.ArchiveMessage(a.ID, msg("<p@x>", "P in a", "p"))
|
||||
|
||||
if n, _ := s.Rethread(a.ID); n != 0 {
|
||||
t.Fatalf("list a has nothing to fix, changed %d", n)
|
||||
}
|
||||
if threadOf(t, s, cb) != cb {
|
||||
t.Fatal("list b must be untouched by rethreading list a")
|
||||
}
|
||||
if n, _ := s.Rethread(b.ID); n != 1 {
|
||||
t.Fatalf("list b: changed %d", n)
|
||||
}
|
||||
if threadOf(t, s, cb) == cb {
|
||||
t.Fatal("list b's child must now follow its parent")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -590,14 +590,52 @@ func scanMessage(row interface{ Scan(...any) error }, withBody bool) (Message, e
|
|||
return m, err
|
||||
}
|
||||
|
||||
// querier is what archiveOne needs from a *sql.DB or a *sql.Tx.
|
||||
type querier interface {
|
||||
Exec(query string, args ...any) (sql.Result, error)
|
||||
QueryRow(query string, args ...any) *sql.Row
|
||||
}
|
||||
|
||||
// ArchiveResult is the outcome for one message of ArchiveBatch.
|
||||
type ArchiveResult struct {
|
||||
ID int64
|
||||
Created bool
|
||||
}
|
||||
|
||||
// ArchiveMessage stores a post. A message already archived in the same list
|
||||
// (same Message-ID) is not stored again: created is false and the existing id
|
||||
// is returned, so MTA retries are harmless.
|
||||
func (s *Store) ArchiveMessage(listID int64, m Message) (id int64, created bool, err error) {
|
||||
return archiveOne(s.db, listID, m)
|
||||
}
|
||||
|
||||
// ArchiveBatch stores many messages in one transaction: all or none. It is much
|
||||
// faster than ArchiveMessage in a loop, because SQLite syncs once per commit.
|
||||
func (s *Store) ArchiveBatch(listID int64, ms []Message) ([]ArchiveResult, error) {
|
||||
if len(ms) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
out := make([]ArchiveResult, 0, len(ms))
|
||||
for _, m := range ms {
|
||||
id, created, err := archiveOne(tx, listID, m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, ArchiveResult{ID: id, Created: created})
|
||||
}
|
||||
return out, tx.Commit()
|
||||
}
|
||||
|
||||
func archiveOne(q querier, listID int64, m Message) (id int64, created bool, err error) {
|
||||
if m.MessageID == "" {
|
||||
return 0, false, errors.New("message id is required")
|
||||
}
|
||||
err = s.db.QueryRow(`SELECT id FROM archive WHERE list_id=? AND message_id=?`, listID, m.MessageID).Scan(&id)
|
||||
err = q.QueryRow(`SELECT id FROM archive WHERE list_id=? AND message_id=?`, listID, m.MessageID).Scan(&id)
|
||||
if err == nil {
|
||||
return id, false, nil
|
||||
}
|
||||
|
|
@ -606,12 +644,12 @@ func (s *Store) ArchiveMessage(listID int64, m Message) (id int64, created bool,
|
|||
}
|
||||
var thread int64
|
||||
if m.InReplyTo != "" {
|
||||
e := s.db.QueryRow(`SELECT thread_id FROM archive WHERE list_id=? AND message_id=?`, listID, m.InReplyTo).Scan(&thread)
|
||||
e := q.QueryRow(`SELECT thread_id FROM archive WHERE list_id=? AND message_id=?`, listID, m.InReplyTo).Scan(&thread)
|
||||
if e != nil && !errors.Is(e, sql.ErrNoRows) {
|
||||
return 0, false, e
|
||||
}
|
||||
}
|
||||
res, err := s.db.Exec(`INSERT INTO archive(list_id, thread_id, message_id, in_reply_to, subject, sender, date, body, size)
|
||||
res, err := q.Exec(`INSERT INTO archive(list_id, thread_id, message_id, in_reply_to, subject, sender, date, body, size)
|
||||
VALUES(?,?,?,?,?,?,?,?,?)`,
|
||||
listID, thread, m.MessageID, m.InReplyTo, m.Subject, m.Sender, m.Date.Unix(), m.Body, len(m.Body))
|
||||
if err != nil {
|
||||
|
|
@ -619,13 +657,102 @@ func (s *Store) ArchiveMessage(listID int64, m Message) (id int64, created bool,
|
|||
}
|
||||
id, _ = res.LastInsertId()
|
||||
if thread == 0 {
|
||||
if _, err := s.db.Exec(`UPDATE archive SET thread_id=id WHERE id=?`, id); err != nil {
|
||||
if _, err := q.Exec(`UPDATE archive SET thread_id=id WHERE id=?`, id); err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
}
|
||||
return id, true, nil
|
||||
}
|
||||
|
||||
// Rethread recomputes the thread of every message of a list from the In-Reply-To
|
||||
// links, and returns how many messages changed thread. Insertion order does not
|
||||
// matter: a message imported before its parent ends up in the parent's thread.
|
||||
// A cycle of replies becomes one thread whose id is the smallest id in the cycle.
|
||||
func (s *Store) Rethread(listID int64) (int, error) {
|
||||
rows, err := s.db.Query(`SELECT id, message_id, in_reply_to, thread_id FROM archive WHERE list_id=?`, listID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
type row struct {
|
||||
id, thread int64
|
||||
replyTo string
|
||||
}
|
||||
var all []row
|
||||
byMsgID := map[string]int64{}
|
||||
for rows.Next() {
|
||||
var r row
|
||||
var msgID string
|
||||
if err := rows.Scan(&r.id, &msgID, &r.replyTo, &r.thread); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
all = append(all, r)
|
||||
byMsgID[msgID] = r.id
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
parent := map[int64]int64{}
|
||||
for _, r := range all {
|
||||
if p, ok := byMsgID[r.replyTo]; ok && r.replyTo != "" {
|
||||
parent[r.id] = p
|
||||
}
|
||||
}
|
||||
roots := map[int64]int64{}
|
||||
root := func(start int64) int64 {
|
||||
if r, ok := roots[start]; ok {
|
||||
return r
|
||||
}
|
||||
path := []int64{start}
|
||||
seen := map[int64]int{start: 0}
|
||||
cur := start
|
||||
var result int64
|
||||
for {
|
||||
p, ok := parent[cur]
|
||||
if !ok {
|
||||
result = cur
|
||||
break
|
||||
}
|
||||
if r, known := roots[p]; known {
|
||||
result = r
|
||||
break
|
||||
}
|
||||
if at, loop := seen[p]; loop { // a cycle: its smallest id is the root
|
||||
result = path[at]
|
||||
for _, id := range path[at:] {
|
||||
result = min(result, id)
|
||||
}
|
||||
break
|
||||
}
|
||||
seen[p] = len(path)
|
||||
path = append(path, p)
|
||||
cur = p
|
||||
}
|
||||
for _, id := range path {
|
||||
roots[id] = result
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
changed := 0
|
||||
for _, r := range all {
|
||||
if want := root(r.id); want != r.thread {
|
||||
if _, err := tx.Exec(`UPDATE archive SET thread_id=? WHERE id=?`, want, r.id); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
changed++
|
||||
}
|
||||
}
|
||||
return changed, tx.Commit()
|
||||
}
|
||||
|
||||
// GetMessage returns an archived message with its body.
|
||||
func (s *Store) GetMessage(id int64) (Message, error) {
|
||||
return scanMessage(s.db.QueryRow(`SELECT `+msgCols+`, body FROM archive WHERE id=?`, id), true)
|
||||
|
|
|
|||
Loading…
Reference in a new issue