parsimony/internal/archive/import.go
tgiachi 93a3edc58f
All checks were successful
ci / admin (push) Successful in 37s
ci / vuln (push) Successful in 48s
ci / go (push) Successful in 3m9s
chore: move the module to git.kmalloc.org and track cmd/parsimony
- Rename the Go module from codeberg.org/squidd/parsimony to
  git.kmalloc.org/squid/parsimony in go.mod and every import.
- Fix .gitignore: the bare `parsimony` pattern also ignored the cmd/parsimony
  folder, so the main package was never committed and a clean checkout could
  not be built. Anchor it to the root (`/parsimony`) and add cmd/parsimony.
2026-11-16 09:40:48 +01:00

106 lines
2.9 KiB
Go

package archive
import (
"errors"
"fmt"
"io"
"time"
"git.kmalloc.org/squid/parsimony/internal/mbox"
"git.kmalloc.org/squid/parsimony/internal/store"
)
const (
// MaxImportMessage is the largest message accepted from an mbox, in bytes.
MaxImportMessage = 25 << 20
// MaxReportedErrors caps the error messages kept in a summary; Failed still counts every one.
MaxReportedErrors = 20
importBatch = 200 // messages per transaction
)
// ImportSummary reports what an import did.
type ImportSummary struct {
Imported int `json:"imported"` // new messages stored
Duplicates int `json:"duplicates"` // already in the archive (same Message-ID)
Failed int `json:"failed"` // unreadable or too large
Errors []string `json:"errors"` // the first MaxReportedErrors failures
}
// ImportMbox archives every message of an mbox into a list. It is idempotent
// (a message is identified by its Message-ID) and works whatever the list's
// archive policy is: the policy only governs live mail. Threads are rebuilt at
// the end, so the order of the messages in the file does not matter. now is the
// date of last resort for a message with neither a Date header nor a dated
// "From " line. On a read error the messages read so far are kept and returned
// in the summary together with the error.
func ImportMbox(st *store.Store, listID int64, r io.Reader, now time.Time) (ImportSummary, error) {
sum := ImportSummary{Errors: []string{}}
var batch []store.Message
fail := func(index int, err error) {
sum.Failed++
if len(sum.Errors) < MaxReportedErrors {
sum.Errors = append(sum.Errors, fmt.Sprintf("message %d: %v", index, err))
}
}
flush := func() error {
results, err := st.ArchiveBatch(listID, batch)
if err != nil {
return err
}
for _, res := range results {
if res.Created {
sum.Imported++
} else {
sum.Duplicates++
}
}
batch = batch[:0]
return nil
}
finish := func(readErr error) (ImportSummary, error) {
if err := flush(); err != nil {
return sum, err
}
if sum.Imported > 0 {
if _, err := st.Rethread(listID); err != nil {
return sum, err
}
}
return sum, readErr
}
rd := mbox.NewReader(r, MaxImportMessage)
for {
m, err := rd.Next()
if err == io.EOF {
return finish(nil)
}
if errors.Is(err, mbox.ErrTooLarge) {
fail(m.Index, fmt.Errorf("too large (limit %d MiB)", MaxImportMessage>>20))
continue
}
if err != nil {
return finish(fmt.Errorf("read mbox: %w", err))
}
fallback := now
if d, ok := m.Date(); ok {
fallback = d
}
en, err := Parse(m.Data, fallback)
if err != nil {
fail(m.Index, err)
continue
}
batch = append(batch, store.Message{
MessageID: en.MessageID, InReplyTo: en.InReplyTo, Subject: en.Subject,
Sender: en.Sender, Date: en.Date, Body: en.Body,
})
if len(batch) >= importBatch {
if err := flush(); err != nil {
return sum, err
}
}
}
}