package archive import ( "errors" "fmt" "io" "time" "git.kmalloc.org/squid/parsimony/internal/mbox" "git.kmalloc.org/squid/parsimony/internal/store" ) const ( // MaxImportMessage is the largest message accepted from an mbox, in bytes. MaxImportMessage = 25 << 20 // MaxReportedErrors caps the error messages kept in a summary; Failed still counts every one. MaxReportedErrors = 20 importBatch = 200 // messages per transaction ) // ImportSummary reports what an import did. type ImportSummary struct { Imported int `json:"imported"` // new messages stored Duplicates int `json:"duplicates"` // already in the archive (same Message-ID) Failed int `json:"failed"` // unreadable or too large Errors []string `json:"errors"` // the first MaxReportedErrors failures } // ImportMbox archives every message of an mbox into a list. It is idempotent // (a message is identified by its Message-ID) and works whatever the list's // archive policy is: the policy only governs live mail. Threads are rebuilt at // the end, so the order of the messages in the file does not matter. now is the // date of last resort for a message with neither a Date header nor a dated // "From " line. On a read error the messages read so far are kept and returned // in the summary together with the error. func ImportMbox(st *store.Store, listID int64, r io.Reader, now time.Time) (ImportSummary, error) { sum := ImportSummary{Errors: []string{}} var batch []store.Message fail := func(index int, err error) { sum.Failed++ if len(sum.Errors) < MaxReportedErrors { sum.Errors = append(sum.Errors, fmt.Sprintf("message %d: %v", index, err)) } } flush := func() error { results, err := st.ArchiveBatch(listID, batch) if err != nil { return err } for _, res := range results { if res.Created { sum.Imported++ } else { sum.Duplicates++ } } batch = batch[:0] return nil } finish := func(readErr error) (ImportSummary, error) { if err := flush(); err != nil { return sum, err } if sum.Imported > 0 { if _, err := st.Rethread(listID); err != nil { return sum, err } } return sum, readErr } rd := mbox.NewReader(r, MaxImportMessage) for { m, err := rd.Next() if err == io.EOF { return finish(nil) } if errors.Is(err, mbox.ErrTooLarge) { fail(m.Index, fmt.Errorf("too large (limit %d MiB)", MaxImportMessage>>20)) continue } if err != nil { return finish(fmt.Errorf("read mbox: %w", err)) } fallback := now if d, ok := m.Date(); ok { fallback = d } en, err := Parse(m.Data, fallback) if err != nil { fail(m.Index, err) continue } batch = append(batch, store.Message{ MessageID: en.MessageID, InReplyTo: en.InReplyTo, Subject: en.Subject, Sender: en.Sender, Date: en.Date, Body: en.Body, }) if len(batch) >= importBatch { if err := flush(); err != nil { return sum, err } } } }