- Rename the Go module from codeberg.org/squidd/parsimony to git.kmalloc.org/squid/parsimony in go.mod and every import. - Fix .gitignore: the bare `parsimony` pattern also ignored the cmd/parsimony folder, so the main package was never committed and a clean checkout could not be built. Anchor it to the root (`/parsimony`) and add cmd/parsimony.
106 lines
2.9 KiB
Go
106 lines
2.9 KiB
Go
package archive
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"time"
|
|
|
|
"git.kmalloc.org/squid/parsimony/internal/mbox"
|
|
"git.kmalloc.org/squid/parsimony/internal/store"
|
|
)
|
|
|
|
const (
|
|
// MaxImportMessage is the largest message accepted from an mbox, in bytes.
|
|
MaxImportMessage = 25 << 20
|
|
// MaxReportedErrors caps the error messages kept in a summary; Failed still counts every one.
|
|
MaxReportedErrors = 20
|
|
|
|
importBatch = 200 // messages per transaction
|
|
)
|
|
|
|
// ImportSummary reports what an import did.
|
|
type ImportSummary struct {
|
|
Imported int `json:"imported"` // new messages stored
|
|
Duplicates int `json:"duplicates"` // already in the archive (same Message-ID)
|
|
Failed int `json:"failed"` // unreadable or too large
|
|
Errors []string `json:"errors"` // the first MaxReportedErrors failures
|
|
}
|
|
|
|
// ImportMbox archives every message of an mbox into a list. It is idempotent
|
|
// (a message is identified by its Message-ID) and works whatever the list's
|
|
// archive policy is: the policy only governs live mail. Threads are rebuilt at
|
|
// the end, so the order of the messages in the file does not matter. now is the
|
|
// date of last resort for a message with neither a Date header nor a dated
|
|
// "From " line. On a read error the messages read so far are kept and returned
|
|
// in the summary together with the error.
|
|
func ImportMbox(st *store.Store, listID int64, r io.Reader, now time.Time) (ImportSummary, error) {
|
|
sum := ImportSummary{Errors: []string{}}
|
|
var batch []store.Message
|
|
|
|
fail := func(index int, err error) {
|
|
sum.Failed++
|
|
if len(sum.Errors) < MaxReportedErrors {
|
|
sum.Errors = append(sum.Errors, fmt.Sprintf("message %d: %v", index, err))
|
|
}
|
|
}
|
|
flush := func() error {
|
|
results, err := st.ArchiveBatch(listID, batch)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, res := range results {
|
|
if res.Created {
|
|
sum.Imported++
|
|
} else {
|
|
sum.Duplicates++
|
|
}
|
|
}
|
|
batch = batch[:0]
|
|
return nil
|
|
}
|
|
finish := func(readErr error) (ImportSummary, error) {
|
|
if err := flush(); err != nil {
|
|
return sum, err
|
|
}
|
|
if sum.Imported > 0 {
|
|
if _, err := st.Rethread(listID); err != nil {
|
|
return sum, err
|
|
}
|
|
}
|
|
return sum, readErr
|
|
}
|
|
|
|
rd := mbox.NewReader(r, MaxImportMessage)
|
|
for {
|
|
m, err := rd.Next()
|
|
if err == io.EOF {
|
|
return finish(nil)
|
|
}
|
|
if errors.Is(err, mbox.ErrTooLarge) {
|
|
fail(m.Index, fmt.Errorf("too large (limit %d MiB)", MaxImportMessage>>20))
|
|
continue
|
|
}
|
|
if err != nil {
|
|
return finish(fmt.Errorf("read mbox: %w", err))
|
|
}
|
|
fallback := now
|
|
if d, ok := m.Date(); ok {
|
|
fallback = d
|
|
}
|
|
en, err := Parse(m.Data, fallback)
|
|
if err != nil {
|
|
fail(m.Index, err)
|
|
continue
|
|
}
|
|
batch = append(batch, store.Message{
|
|
MessageID: en.MessageID, InReplyTo: en.InReplyTo, Subject: en.Subject,
|
|
Sender: en.Sender, Date: en.Date, Body: en.Body,
|
|
})
|
|
if len(batch) >= importBatch {
|
|
if err := flush(); err != nil {
|
|
return sum, err
|
|
}
|
|
}
|
|
}
|
|
}
|